From 474f37b5409bc9cad99cf8919b122869c7ec1f3e Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sat, 8 Aug 2026 00:08:18 -0400 Subject: [PATCH 1/5] feat(extensions): support custom syntax grammars --- .changeset/extension-syntax-languages.md | 5 + README.md | 4 +- docs/extension-architecture.md | 8 +- docs/extensions.md | 47 ++- scripts/check-pack.ts | 7 +- src/core/fileLanguage.ts | 49 ++- src/extension-api/index.ts | 2 + src/extension-api/types.ts | 22 +- src/extensions/apply.test.ts | 313 ++++++++++++++++++ src/extensions/apply.ts | 133 +++++++- src/extensions/index.ts | 4 + src/extensions/publicApiRobustness.test.ts | 40 +++ src/extensions/runExtension.test.ts | 66 ++++ src/extensions/runExtension.ts | 23 +- src/extensions/types.ts | 11 + src/ui/diff/pierre.ts | 107 +++--- src/ui/diff/useHighlightedDiff.test.ts | 9 + src/ui/diff/useHighlightedDiff.ts | 2 +- .../content/docs/docs/extend/extension-api.md | 26 +- 19 files changed, 797 insertions(+), 81 deletions(-) create mode 100644 .changeset/extension-syntax-languages.md diff --git a/.changeset/extension-syntax-languages.md b/.changeset/extension-syntax-languages.md new file mode 100644 index 000000000..04563ec41 --- /dev/null +++ b/.changeset/extension-syntax-languages.md @@ -0,0 +1,5 @@ +--- +"hunkdiff": minor +--- + +Let extensions register lazy custom syntax grammars for file highlighting. diff --git a/README.md b/README.md index ddaf6c024..6725b2e1f 100644 --- a/README.md +++ b/README.md @@ -210,8 +210,8 @@ repository's `.hunk/extensions/` (after you explicitly trust that repository), and from `--extension ` for development. `--no-extensions` turns those off for one run; Hunk's own bundled backends (Git, Jujutsu, and Sapling) stay loaded. -A Phase 1 extension can contribute themes and file-extension → language -mappings, add a VCS backend, rewrite the changeset before review (collapse +An extension can contribute themes, lazy syntax grammars, and file-extension → +language mappings, add a VCS backend, rewrite the changeset before review (collapse lockfiles, reorder files by review priority), replace the file-navigation sidebar with its own React component, react to lifecycle events, and show transient messages: diff --git a/docs/extension-architecture.md b/docs/extension-architecture.md index 95e20ad7d..2ae542f5d 100644 --- a/docs/extension-architecture.md +++ b/docs/extension-architecture.md @@ -46,12 +46,16 @@ load issue and costs only that extension. The rules themselves are stated in ## One registry, one apply path -Registrations (themes, file languages, VCS adapters, changeset transforms, +Registrations (themes, syntax languages, file languages, VCS adapters, changeset transforms, sidebar views, commands, lifecycle/UI events, and inter-extension bus listeners) collect into one `ExtensionRegistry` (`src/extensions/types.ts`) and are resolved/applied through `src/extensions/apply.ts` on both startup and reload. A factory that throws is rolled back to its pre-run registration counts -(`runExtension.ts`); failures cost a warning, not the session. +(`runExtension.ts`); failures cost a warning, not the session. Syntax-language +loaders are registered with Pierre only at this apply boundary, preserving +factory rollback. Pierre owns the process-wide grammar cache, so an extension +reload may add a new syntax id but changing or removing an applied grammar needs +a Hunk restart. ## Host-served runtime modules diff --git a/docs/extensions.md b/docs/extensions.md index 4b7dad9fd..68283fed5 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -188,7 +188,7 @@ cannot mutate the registry mid-session. ### `hunk.apiVersion` -The API generation this Hunk speaks (currently `2`). Branch on it if you want +The API generation this Hunk speaks (currently `3`). Branch on it if you want one file to support several Hunk versions. ### `hunk.registerTheme(theme)` @@ -211,10 +211,53 @@ built-in id. Config-defined themes always win over extension themes for the same id; the loser is reported as a startup notice. Extension themes appear in the selector after config themes, in load order. +### `hunk.registerSyntaxLanguage(language, loader)` + +Register a Shiki-compatible TextMate grammar under a highlighting language id. +The loader is lazy: Hunk calls it only when Pierre first highlights that +language. It resolves to an ES-module-shaped object whose `default` export is a +non-empty grammar array. Every grammar needs at least `name` and `scopeName`; +normal TextMate fields such as `patterns` and `repository` pass through to +Shiki. + +```ts +hunk.registerSyntaxLanguage("example-lang", async () => ({ + default: [ + { + name: "example-lang", + scopeName: "source.example-lang", + patterns: [{ match: "\\b(example|language)\\b", name: "keyword.control.example-lang" }], + repository: {}, + }, + ], +})); +hunk.registerFileLanguage("example", "example-lang"); +``` + +A folder extension can keep a generated grammar in a helper module or install a +language package in its own `node_modules`, then pass its dynamic import directly +(for example, `() => import("@shikijs/langs/odin")`). Import Pierre through neither +the extension nor that helper: Hunk forwards the loader to the host's own Pierre +instance so the grammar reaches the highlighter that renders the review. + +Syntax ids are trimmed but remain case-sensitive. `text` and `ansi` are +reserved. The first extension to register another id wins; later claims are +skipped with an attributed notice. When registered before Pierre first resolves +it, a custom id takes precedence over a Pierre-bundled grammar with the same id, +so choose a new id unless replacing the built-in grammar is deliberate. + +Pierre's grammar registry lasts for the Hunk process and cannot safely replace +or unregister a loader. Reloading the same extension is idempotent, but grammar +changes, removals, and replacing a grammar already used by this process require +restarting Hunk. A loader that rejects, returns an invalid module, or supplies a +grammar Shiki cannot attach produces one attributed warning when first used, +and that file falls back to unhighlighted text. + ### `hunk.registerFileLanguage(extension, language)` Map a file extension to a syntax-highlighting language. The extension may be -written with or without a leading dot and is lowercased. +written with or without a leading dot and is lowercased. Use +`registerSyntaxLanguage` first when the language is not bundled with Pierre. ```ts hunk.registerFileLanguage(".zig", "zig"); diff --git a/scripts/check-pack.ts b/scripts/check-pack.ts index 217ae74b9..60b8a16be 100644 --- a/scripts/check-pack.ts +++ b/scripts/check-pack.ts @@ -28,6 +28,7 @@ import type { ExtensionFileViewSourceRange, ExtensionPaintTheme, ExtensionReviewSelection, + ExtensionSyntaxLanguageLoader, ExtensionVcsAdapter, ExtensionVcsDiffInput, ExtensionVcsLoadContext, @@ -49,6 +50,10 @@ export default function (hunk: HunkExtensionAPI) { syntaxScopes: { "keyword.operator": "#7fd1ff" }, }; hunk.registerTheme(theme); + const syntaxLoader: ExtensionSyntaxLanguageLoader = async () => ({ + default: [{ name: "hunk-pack-fixture", scopeName: "source.hunk-pack-fixture" }], + }); + hunk.registerSyntaxLanguage("hunk-pack-fixture", syntaxLoader); hunk.registerFileLanguage(".zig", "zig"); const renderRow = (props: ExtensionFileViewRowComponentProps) => { @@ -317,7 +322,7 @@ const extensionTypes = readFileSync( path.join(repoRoot, "dist", "npm", "extension", "extension-api", "types.d.ts"), "utf8", ); -if (/^\s*import\b/m.test(extensionTypes)) { +if (/^\s*import\b/m.test(extensionTypes) || /\bimport\s*\(/m.test(extensionTypes)) { throw new Error("The public extension-api/types declaration must remain import-free."); } for (const removedType of [ diff --git a/src/core/fileLanguage.ts b/src/core/fileLanguage.ts index 6a70b4307..8a7e42bee 100644 --- a/src/core/fileLanguage.ts +++ b/src/core/fileLanguage.ts @@ -1,8 +1,12 @@ import { + getCustomExtensionsVersion, getFiletypeFromFileName, - setCustomExtension, + registerCustomLanguage, + replaceCustomExtensions, + type LanguageRegistration, type SupportedLanguages, } from "@pierre/diffs"; +import type { ExtensionSyntaxLanguageLoader } from "../extension-api/types"; // Pierre omits these TypeScript extensions, so register them before lookups or rendering. const HUNK_CUSTOM_EXTENSIONS: Record = { @@ -10,9 +14,7 @@ const HUNK_CUSTOM_EXTENSIONS: Record = { cts: "typescript", }; -for (const [extension, language] of Object.entries(HUNK_CUSTOM_EXTENSIONS)) { - setCustomExtension(extension, language); -} +replaceCustomExtensions(getCustomExtensionsVersion() + 1, HUNK_CUSTOM_EXTENSIONS); /** * Extensions Hunk itself registers, in Pierre's dotless lowercase form. @@ -25,15 +27,36 @@ export const BUILT_IN_FILE_LANGUAGE_EXTENSIONS: ReadonlySet = new Set( Object.keys(HUNK_CUSTOM_EXTENSIONS), ); -/** - * Map one dotless, lowercased file extension to a highlight language. - * - * Pierre's language union is closed, but extensions supply plain strings; an - * unknown language simply fails to match a grammar at render time, which is a - * better failure than refusing the registration outright. - */ -export function registerFileLanguage(extension: string, language: string) { - setCustomExtension(extension, language as SupportedLanguages); +const syntaxLanguageFailureReporters = new Map void>(); + +/** Register one lazy extension grammar with Pierre's process-wide highlighter. */ +export function registerSyntaxLanguage( + language: string, + loader: ExtensionSyntaxLanguageLoader, + reportFailure?: (error: unknown) => void, +) { + registerCustomLanguage(language, loader as () => Promise<{ default: LanguageRegistration[] }>); + if (reportFailure) { + syntaxLanguageFailureReporters.set(language, reportFailure); + } +} + +/** Attribute a highlight failure when its language came from an extension. */ +export function reportSyntaxLanguageFailure(language: string | undefined, error: unknown) { + if (language) { + syntaxLanguageFailureReporters.get(language)?.(error); + } +} + +/** Replace extension mappings while preserving Hunk's own protected mappings. */ +export function replaceExtensionFileLanguages( + mappings: ReadonlyArray<{ extension: string; language: string }>, +) { + const desired: Record = { ...HUNK_CUSTOM_EXTENSIONS }; + for (const { extension, language } of mappings) { + desired[extension] = language as SupportedLanguages; + } + replaceCustomExtensions(getCustomExtensionsVersion() + 1, desired); } export { getFiletypeFromFileName }; diff --git a/src/extension-api/index.ts b/src/extension-api/index.ts index e7ddea1af..ffeed6e74 100644 --- a/src/extension-api/index.ts +++ b/src/extension-api/index.ts @@ -85,6 +85,8 @@ export type { ExtensionSidebarTheme, ExtensionSidebarView, ExtensionSidebarViewProps, + ExtensionSyntaxGrammar, + ExtensionSyntaxLanguageLoader, ExtensionThemeConfig, ExtensionVcsAdapter, ExtensionVcsDetection, diff --git a/src/extension-api/types.ts b/src/extension-api/types.ts index e0bebe63e..351f85cda 100644 --- a/src/extension-api/types.ts +++ b/src/extension-api/types.ts @@ -21,7 +21,7 @@ * Extensions can branch on `hunk.apiVersion` so a newer Hunk can keep loading * older extensions without guessing at their expectations. */ -export const HUNK_EXTENSION_API_VERSION = 2; +export const HUNK_EXTENSION_API_VERSION = 3; export type HunkExtensionApiVersion = typeof HUNK_EXTENSION_API_VERSION; export type ExtensionNotifyType = "info" | "warning" | "error"; @@ -479,6 +479,21 @@ export interface NamedCustomThemeConfig extends CustomThemeConfig { */ export type ExtensionThemeConfig = NamedCustomThemeConfig; +/* -------------------------------------------------------------------------- */ +/* Syntax languages */ +/* -------------------------------------------------------------------------- */ + +/** Minimum public shape of one Shiki-compatible TextMate grammar. */ +export interface ExtensionSyntaxGrammar { + readonly name: string; + readonly scopeName: string; +} + +/** Lazy ES module containing the grammar registrations for one syntax language. */ +export type ExtensionSyntaxLanguageLoader< + Grammar extends ExtensionSyntaxGrammar = ExtensionSyntaxGrammar, +> = () => Promise<{ readonly default: readonly Grammar[] }>; + /* -------------------------------------------------------------------------- */ /* VCS adapters */ /* -------------------------------------------------------------------------- */ @@ -1444,6 +1459,11 @@ export interface HunkExtensionAPI { readonly apiVersion: HunkExtensionApiVersion; /** Contribute one selectable theme. */ registerTheme(theme: ExtensionThemeConfig): void; + /** Register a lazy Shiki/TextMate grammar under one highlight-language id. */ + registerSyntaxLanguage( + language: string, + loader: ExtensionSyntaxLanguageLoader, + ): void; /** Map one file extension (with or without a leading dot) to a highlight language. */ registerFileLanguage(extension: string, language: string): void; /** Contribute one additional VCS backend. */ diff --git a/src/extensions/apply.test.ts b/src/extensions/apply.test.ts index cab8c482d..9d07adb17 100644 --- a/src/extensions/apply.test.ts +++ b/src/extensions/apply.test.ts @@ -3,15 +3,21 @@ import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { createTestDiffFile } from "../../test/helpers/diff-helpers"; +import { createTestCustomThemes } from "../../test/helpers/theme-helpers"; +import { parseDiffFromFile, resolveLanguage } from "@pierre/diffs"; import { HUNK_CORE_VCS_DETECTION_PRIORITY, HUNK_DEFAULT_VCS_DETECTION_PRIORITY, } from "../extension-api/types"; +import { getFiletypeFromFileName } from "../core/fileLanguage"; import type { Changeset, DiffFile } from "../core/types"; import type { VcsAdapter } from "../core/vcs/types"; +import { buildStackRows, loadHighlightedDiff } from "../ui/diff/pierre"; +import { resolveTheme } from "../ui/themes"; import { applyExtensionChangesetTransforms, applyExtensionFileLanguages, + applyExtensionSyntaxLanguages, createExtensionApplyNotices, reportExtensionApplyIssues, createUnknownVcsNotice, @@ -76,6 +82,297 @@ function createTestVcsAdapter(id: string): VcsAdapter { return { id, name: id, detect: () => null, operations: {} }; } +/** Build one lazy TextMate grammar module accepted by the public contract. */ +function createTestSyntaxLoader(language: string, onLoad?: () => void) { + return async () => { + onLoad?.(); + return { + default: [ + { + name: language, + scopeName: `source.${language}`, + patterns: [], + repository: {}, + }, + ], + }; + }; +} + +describe("extension syntax languages", () => { + test("registers a grammar lazily in Pierre's host-owned language registry", async () => { + const { result } = createTestLoadResult(); + let loadCount = 0; + const language = "hunk-lazy-syntax-fixture"; + result.registry.extensions.push({ + id: "syntax-pack", + sourcePath: "/extensions/syntax-pack.ts", + origin: "global", + }); + result.registry.syntaxLanguages.push({ + extensionId: "syntax-pack", + language, + loader: createTestSyntaxLoader(language, () => (loadCount += 1)), + }); + + expect(applyExtensionSyntaxLanguages(result.registry, result.context)).toEqual([]); + expect(loadCount).toBe(0); + + const resolved = await resolveLanguage(language); + expect(loadCount).toBe(1); + expect(resolved.name).toBe(language); + expect(resolved.data[0]?.scopeName).toBe(`source.${language}`); + }); + + test("highlights mapped files with extension-contributed grammar scopes", async () => { + const { result } = createTestLoadResult(); + const language = "hunk-rendered-syntax-fixture"; + result.registry.syntaxLanguages.push({ + extensionId: "rendered-syntax", + language, + loader: async () => ({ + default: [ + { + name: language, + scopeName: `source.${language}`, + patterns: [ + { + match: "\\bmagic\\b", + name: `keyword.control.${language}`, + }, + ], + repository: {}, + }, + ], + }), + }); + result.registry.fileLanguages.push({ + extensionId: "rendered-syntax", + extension: "hunkrenderedfixture", + language, + }); + + expect(applyExtensionSyntaxLanguages(result.registry, result.context)).toEqual([]); + expect(applyExtensionFileLanguages(result.registry)).toEqual([]); + + const path = "example.hunkrenderedfixture"; + const metadata = parseDiffFromFile( + { name: path, contents: "ordinary\n", cacheKey: "custom-syntax-before" }, + { name: path, contents: "magic\n", cacheKey: "custom-syntax-after" }, + { context: 3 }, + true, + ); + const file: DiffFile = { + id: "rendered-custom-syntax", + path, + patch: "", + language: getFiletypeFromFileName(path), + stats: { additions: 1, deletions: 1 }, + metadata, + agent: null, + }; + const theme = resolveTheme( + "custom", + null, + createTestCustomThemes({ + base: "github-dark-default", + syntaxScopes: { [`keyword.control.${language}`]: "#123456" }, + }), + ); + const highlighted = await loadHighlightedDiff(file, theme); + const spans = buildStackRows(file, highlighted, theme) + .filter((row) => row.type === "stack-line" && row.cell.kind === "addition") + .flatMap((row) => (row.type === "stack-line" ? row.cell.spans : [])); + + expect(file.language).toBe(language); + expect(spans.find((span) => span.text === "magic")?.fg).toBe("#123456"); + }); + + test("keeps the first grammar registration and attributes later conflicts", async () => { + const { result } = createTestLoadResult(); + const language = "hunk-duplicate-syntax-fixture"; + result.registry.syntaxLanguages.push( + { + extensionId: "first", + language, + loader: createTestSyntaxLoader(language), + }, + { + extensionId: "second", + language, + loader: createTestSyntaxLoader("ignored-syntax-fixture"), + }, + ); + + expect(applyExtensionSyntaxLanguages(result.registry, result.context)).toEqual([ + { + extensionId: "second", + message: `Skipped syntax language "${language}" from extension second • extension first registered it first`, + }, + ]); + expect((await resolveLanguage(language)).data[0]?.name).toBe(language); + }); + + test("reserves Pierre's plain-text language ids", () => { + const { result } = createTestLoadResult(); + result.registry.syntaxLanguages.push( + { extensionId: "syntax-pack", language: "text", loader: createTestSyntaxLoader("text") }, + { extensionId: "syntax-pack", language: "ansi", loader: createTestSyntaxLoader("ansi") }, + ); + + expect(applyExtensionSyntaxLanguages(result.registry, result.context)).toEqual([ + { + extensionId: "syntax-pack", + message: 'Skipped syntax language "text" from extension syntax-pack • Hunk reserves it', + }, + { + extensionId: "syntax-pack", + message: 'Skipped syntax language "ansi" from extension syntax-pack • Hunk reserves it', + }, + ]); + }); + + test("reports malformed lazy modules once and leaves Pierre's failure retryable", async () => { + const { result, notices } = createTestLoadResult(); + const language = "hunk-invalid-syntax-fixture"; + result.registry.syntaxLanguages.push({ + extensionId: "broken-syntax", + language, + loader: async () => ({ + default: [{ name: "", scopeName: "" }], + }), + }); + + expect(applyExtensionSyntaxLanguages(result.registry, result.context)).toEqual([]); + await expect(resolveLanguage(language)).rejects.toThrow("loader must resolve"); + await expect(resolveLanguage(language)).rejects.toThrow("loader must resolve"); + expect(notices).toHaveLength(1); + expect(notices[0]).toContain( + `Failed to highlight syntax language "${language}" from extension broken-syntax`, + ); + }); + + test("attributes grammar attachment failures before falling back to text", async () => { + const { result, notices } = createTestLoadResult(); + const language = "hunk-broken-render-syntax-fixture"; + result.registry.syntaxLanguages.push({ + extensionId: "broken-render-syntax", + language, + loader: async () => ({ + default: [ + { + name: language, + scopeName: `source.${language}`, + patterns: [null], + repository: {}, + }, + ], + }), + }); + applyExtensionSyntaxLanguages(result.registry, result.context); + + const path = "broken.hunkrenderedfixture"; + const metadata = parseDiffFromFile( + { name: path, contents: "before\n", cacheKey: "broken-syntax-before" }, + { name: path, contents: "after\n", cacheKey: "broken-syntax-after" }, + { context: 3 }, + true, + ); + const file: DiffFile = { + id: "broken-rendered-custom-syntax", + path, + patch: "", + language, + stats: { additions: 1, deletions: 1 }, + metadata, + agent: null, + }; + + const recoveryPath = "recovery.ex"; + const recoveryMetadata = parseDiffFromFile( + { name: recoveryPath, contents: "", cacheKey: "syntax-recovery-before" }, + { + name: recoveryPath, + contents: "def recovered do\n :ok\nend\n", + cacheKey: "syntax-recovery-after", + }, + { context: 3 }, + true, + ); + const recoveryFile: DiffFile = { + id: "syntax-recovery", + path: recoveryPath, + patch: "", + language: "elixir", + stats: { additions: 3, deletions: 0 }, + metadata: recoveryMetadata, + agent: null, + }; + const recoveryTheme = resolveTheme( + "custom", + null, + createTestCustomThemes({ + base: "github-dark-default", + syntaxScopes: { "keyword.control.hunk-recovery-theme": "#13579b" }, + }), + ); + const [highlighted, recoveryHighlight] = await Promise.all([ + loadHighlightedDiff(file, recoveryTheme), + loadHighlightedDiff(recoveryFile, recoveryTheme), + ]); + + expect(highlighted.additionLines.length).toBeGreaterThan(0); + expect(notices).toHaveLength(1); + expect(notices[0]).toContain( + `Failed to highlight syntax language "${language}" from extension broken-render-syntax`, + ); + const recoverySpans = buildStackRows(recoveryFile, recoveryHighlight, recoveryTheme) + .filter((row) => row.type === "stack-line" && row.cell.kind === "addition") + .flatMap((row) => (row.type === "stack-line" ? row.cell.spans : [])); + + expect(recoverySpans.some((span) => span.text.includes("def") && span.fg !== undefined)).toBe( + true, + ); + }); + + test("reapplying one source is idempotent while another source cannot replace it", () => { + const language = "hunk-reload-syntax-fixture"; + const first = createTestLoadResult().result; + first.registry.extensions.push({ + id: "syntax-pack", + sourcePath: "/extensions/first.ts", + origin: "global", + }); + first.registry.syntaxLanguages.push({ + extensionId: "syntax-pack", + language, + loader: createTestSyntaxLoader(language), + }); + + expect(applyExtensionSyntaxLanguages(first.registry, first.context)).toEqual([]); + expect(applyExtensionSyntaxLanguages(first.registry, first.context)).toEqual([]); + + const second = createTestLoadResult().result; + second.registry.extensions.push({ + id: "syntax-pack", + sourcePath: "/extensions/second.ts", + origin: "global", + }); + second.registry.syntaxLanguages.push({ + extensionId: "syntax-pack", + language, + loader: createTestSyntaxLoader("replacement-syntax-fixture"), + }); + + expect(applyExtensionSyntaxLanguages(second.registry, second.context)).toEqual([ + { + extensionId: "syntax-pack", + message: `Skipped syntax language "${language}" from extension syntax-pack • already loaded by extension syntax-pack; restart Hunk to replace it`, + }, + ]); + }); +}); + describe("extension file languages", () => { test("registers extension mappings and skips built-in ones", () => { const { result } = createTestLoadResult(); @@ -105,6 +402,22 @@ describe("extension file languages", () => { expect(applyExtensionFileLanguages(result.registry)).toEqual([]); expect(getFiletypeFromFileName("sample.hunkfixture")).toBe("ruby"); }); + + test("removes mappings retired by an extension reload", async () => { + const { getFiletypeFromFileName } = await import("../core/fileLanguage"); + const { result } = createTestLoadResult(); + result.registry.fileLanguages.push({ + extensionId: "temporary", + extension: "hunkretiredfixture", + language: "python", + }); + + applyExtensionFileLanguages(result.registry); + expect(getFiletypeFromFileName("sample.hunkretiredfixture")).toBe("python"); + + applyExtensionFileLanguages(createTestLoadResult().result.registry); + expect(getFiletypeFromFileName("sample.hunkretiredfixture")).toBe("text"); + }); }); describe("extension VCS adapters", () => { diff --git a/src/extensions/apply.ts b/src/extensions/apply.ts index 192141104..b96d47601 100644 --- a/src/extensions/apply.ts +++ b/src/extensions/apply.ts @@ -1,4 +1,8 @@ -import { BUILT_IN_FILE_LANGUAGE_EXTENSIONS, registerFileLanguage } from "../core/fileLanguage"; +import { + BUILT_IN_FILE_LANGUAGE_EXTENSIONS, + registerSyntaxLanguage, + replaceExtensionFileLanguages, +} from "../core/fileLanguage"; import type { StartupNotice } from "../core/startupNotice"; import type { Changeset } from "../core/types"; import { detectVcs, getDefaultVcsAdapter, isVcsId, resolveVcsAdapters } from "../core/vcs"; @@ -34,6 +38,123 @@ function describeError(error: unknown) { return String(error); } +const RESERVED_SYNTAX_LANGUAGE_IDS = new Set(["text", "ansi"]); + +interface AppliedSyntaxLanguage { + extensionId: string; + sourcePath: string; +} + +// Pierre exposes no supported replace/unregister operation, so claims live for +// the process and extension reloads may only add previously unseen language ids. +const appliedSyntaxLanguages = new Map(); + +/** Report whether one lazy module contains usable TextMate grammar records. */ +function isSyntaxLanguageModule( + value: unknown, +): value is Awaited> { + const module = value as { default?: unknown } | null; + return ( + typeof module === "object" && + module !== null && + Array.isArray(module.default) && + module.default.length > 0 && + module.default.every( + (grammar) => + typeof grammar === "object" && + grammar !== null && + typeof (grammar as { name?: unknown }).name === "string" && + (grammar as { name: string }).name.trim().length > 0 && + typeof (grammar as { scopeName?: unknown }).scopeName === "string" && + (grammar as { scopeName: string }).scopeName.trim().length > 0, + ) + ); +} + +/** Add extension-contributed lazy grammars to Pierre's process-wide registry. */ +export function applyExtensionSyntaxLanguages( + registry: ExtensionRegistry, + context?: ExtensionContext, +): ExtensionApplyIssue[] { + const issues: ExtensionApplyIssue[] = []; + const claimedThisPass = new Map(); + + for (const { extensionId, language, loader } of registry.syntaxLanguages) { + if (RESERVED_SYNTAX_LANGUAGE_IDS.has(language)) { + issues.push({ + extensionId, + message: `Skipped syntax language "${language}" from extension ${extensionId} • Hunk reserves it`, + }); + continue; + } + + const passOwner = claimedThisPass.get(language); + if (passOwner) { + issues.push({ + extensionId, + message: `Skipped syntax language "${language}" from extension ${extensionId} • extension ${passOwner} registered it first`, + }); + continue; + } + claimedThisPass.set(language, extensionId); + + const sourcePath = + registry.extensions.find((extension) => extension.id === extensionId)?.sourcePath ?? + extensionId; + const applied = appliedSyntaxLanguages.get(language); + if (applied) { + // Reapplying the same extension is idempotent. Pierre cannot replace the + // old loader, so a changed grammar takes effect after Hunk restarts. + if (applied.extensionId === extensionId && applied.sourcePath === sourcePath) { + continue; + } + issues.push({ + extensionId, + message: `Skipped syntax language "${language}" from extension ${extensionId} • already loaded by extension ${applied.extensionId}; restart Hunk to replace it`, + }); + continue; + } + + let reportedFailure = false; + const reportFailure = (error: unknown) => { + if (reportedFailure) { + return; + } + reportedFailure = true; + context?.notify( + `Failed to highlight syntax language "${language}" from extension ${extensionId} • ${sanitizeTerminalLine(describeError(error))}`, + "warning", + ); + }; + const attributedLoader: typeof loader = async () => { + try { + const module = await loader(); + if (!isSyntaxLanguageModule(module)) { + throw new Error( + "loader must resolve to { default: Grammar[] } with non-empty name and scopeName fields", + ); + } + return module; + } catch (error) { + reportFailure(error); + throw error; + } + }; + + try { + registerSyntaxLanguage(language, attributedLoader, reportFailure); + appliedSyntaxLanguages.set(language, { extensionId, sourcePath }); + } catch (error) { + issues.push({ + extensionId, + message: `Skipped syntax language "${language}" from extension ${extensionId} • ${sanitizeTerminalLine(describeError(error))}`, + }); + } + } + + return issues; +} + /** * Register every extension-contributed file-extension → language mapping. * @@ -44,6 +165,7 @@ function describeError(error: unknown) { */ export function applyExtensionFileLanguages(registry: ExtensionRegistry): ExtensionApplyIssue[] { const issues: ExtensionApplyIssue[] = []; + const mappings: Array<{ extension: string; language: string }> = []; for (const { extensionId, extension, language } of registry.fileLanguages) { if (BUILT_IN_FILE_LANGUAGE_EXTENSIONS.has(extension)) { @@ -54,9 +176,10 @@ export function applyExtensionFileLanguages(registry: ExtensionRegistry): Extens continue; } - registerFileLanguage(extension, language); + mappings.push({ extension, language }); } + replaceExtensionFileLanguages(mappings); return issues; } @@ -256,7 +379,8 @@ export function applyExtensionRegistrations( return { vcsAdapters: [], issues: [] }; } - const languageIssues = applyExtensionFileLanguages(result.registry); + const syntaxLanguageIssues = applyExtensionSyntaxLanguages(result.registry, result.context); + const fileLanguageIssues = applyExtensionFileLanguages(result.registry); const vcs = resolveExtensionVcsAdapters(result.registry); // Resolved again where the UI consumes them; consulted here so skipped // duplicate registrations surface through the same notice path as every @@ -267,7 +391,8 @@ export function applyExtensionRegistrations( return { vcsAdapters: vcs.adapters, issues: [ - ...languageIssues, + ...syntaxLanguageIssues, + ...fileLanguageIssues, ...vcs.issues, ...sidebars.issues, ...fileViews.issues, diff --git a/src/extensions/index.ts b/src/extensions/index.ts index fa2c26a98..d33ba5394 100644 --- a/src/extensions/index.ts +++ b/src/extensions/index.ts @@ -2,6 +2,7 @@ export { applyExtensionChangesetTransforms, applyExtensionFileLanguages, applyExtensionRegistrations, + applyExtensionSyntaxLanguages, createExtensionApplyNotices, reportExtensionApplyIssues, resolveDetectedVcsIdWithExtensions, @@ -76,6 +77,8 @@ export type { ExtensionNotifyType, ExtensionOrigin, ExtensionRegistry, + ExtensionSyntaxGrammar, + ExtensionSyntaxLanguageLoader, ExtensionThemeConfig, HunkExtensionAPI, HunkExtensionApiVersion, @@ -84,6 +87,7 @@ export type { RegisteredEventHandler, RegisteredFileLanguage, RegisteredSidebarView, + RegisteredSyntaxLanguage, RegisteredTheme, RegisteredVcsAdapter, SessionReloadReason, diff --git a/src/extensions/publicApiRobustness.test.ts b/src/extensions/publicApiRobustness.test.ts index 434760b45..97366b3e8 100644 --- a/src/extensions/publicApiRobustness.test.ts +++ b/src/extensions/publicApiRobustness.test.ts @@ -131,6 +131,46 @@ describe("registerTheme with junk", () => { }); }); +describe("registerSyntaxLanguage with junk", () => { + test("refuses unusable language ids and loaders as load issues", () => { + for (const [language, loader] of [ + ["", async () => ({ default: [] })], + [" ", async () => ({ default: [] })], + [null, async () => ({ default: [] })], + ["demo", null], + ["demo", {}], + [undefined, undefined], + ] as Array<[unknown, unknown]>) { + const { registry, issues } = loadFactory( + (hunk: { registerSyntaxLanguage: (language: unknown, loader: unknown) => void }) => { + hunk.registerSyntaxLanguage(language, loader); + }, + ); + + expect(issues).toHaveLength(1); + expect(issues[0]?.extensionId).toBe("fuzz-ext"); + expect(registry.syntaxLanguages).toEqual([]); + expect(registry.extensions).toEqual([]); + } + }); + + test("preserves a case-sensitive language id while trimming whitespace", () => { + const loader = async () => ({ + default: [{ name: "DemoLang", scopeName: "source.DemoLang" }], + }); + const { registry, issues } = loadFactory( + (hunk: { registerSyntaxLanguage: (language: string, candidate: typeof loader) => void }) => { + hunk.registerSyntaxLanguage(" DemoLang ", loader); + }, + ); + + expect(issues).toEqual([]); + expect(registry.syntaxLanguages).toEqual([ + { extensionId: "fuzz-ext", language: "DemoLang", loader }, + ]); + }); +}); + describe("registerFileLanguage with junk", () => { test("refuses unusable extensions and languages as load issues", () => { for (const [extension, language] of [ diff --git a/src/extensions/runExtension.test.ts b/src/extensions/runExtension.test.ts index d5328cad5..6a991b527 100644 --- a/src/extensions/runExtension.test.ts +++ b/src/extensions/runExtension.test.ts @@ -100,6 +100,72 @@ describe("runExtensionFactory", () => { }); }); +describe("registerSyntaxLanguage", () => { + test("collects a trimmed language id and lazy grammar loader", async () => { + const registry = createEmptyExtensionRegistry(); + const issues: ExtensionLoadIssue[] = []; + const loader = async () => ({ + default: [{ name: "demo-syntax", scopeName: "source.demo-syntax" }], + }); + + runExtensionFactory({ + metadata: bundledMetadata("syntax"), + registry, + issues, + factory: (hunk) => { + hunk.registerSyntaxLanguage(" demo-syntax ", loader); + }, + }); + + expect(issues).toEqual([]); + expect(registry.syntaxLanguages).toEqual([ + { extensionId: "syntax", language: "demo-syntax", loader }, + ]); + expect(await registry.syntaxLanguages[0]?.loader()).toEqual(await loader()); + }); + + test("rolls grammar registration back when the factory fails", () => { + const registry = createEmptyExtensionRegistry(); + const issues: ExtensionLoadIssue[] = []; + + runExtensionFactory({ + metadata: bundledMetadata("broken-syntax"), + registry, + issues, + factory: (hunk) => { + hunk.registerSyntaxLanguage("broken-syntax", async () => ({ + default: [{ name: "broken-syntax", scopeName: "source.broken-syntax" }], + })); + throw new Error("boom"); + }, + }); + + expect(registry.syntaxLanguages).toEqual([]); + expect(issues.map((issue) => issue.message)).toEqual(["boom"]); + }); + + test("seals grammar registration with the rest of the factory API", () => { + const registry = createEmptyExtensionRegistry(); + let register: + | ((language: string, loader: () => Promise<{ default: never[] }>) => void) + | undefined; + + runExtensionFactory({ + metadata: bundledMetadata("late-syntax"), + registry, + issues: [], + factory: (hunk) => { + register = hunk.registerSyntaxLanguage; + }, + }); + + expect(() => register?.("late-syntax", async () => ({ default: [] }))).toThrow( + "late-syntax: hunk.registerSyntaxLanguage() can only be called while the extension is loading.", + ); + expect(registry.syntaxLanguages).toEqual([]); + }); +}); + describe("registerSidebarView", () => { test("collects a valid view tagged with the owning extension", () => { const registry = createEmptyExtensionRegistry(); diff --git a/src/extensions/runExtension.ts b/src/extensions/runExtension.ts index 3a20c7bed..76d96a161 100644 --- a/src/extensions/runExtension.ts +++ b/src/extensions/runExtension.ts @@ -13,6 +13,7 @@ import { type ExtensionRegistry, type ExtensionSidebarView, type ExtensionFileView, + type ExtensionSyntaxLanguageLoader, type ExtensionThemeConfig, type ExtensionVcsAdapter, type HunkExtensionAPI, @@ -51,6 +52,11 @@ function normalizeFileExtension(extension: string) { return normalized; } +/** Normalize one highlighting id while preserving its case-sensitive spelling. */ +function normalizeSyntaxLanguageId(language: unknown, method: string) { + return assertNonEmptyString(language, `${method} requires a non-empty language.`).trim(); +} + /** Reject registrations that would leave the registry holding unusable entries. */ function assertNonEmptyString(value: unknown, message: string) { if (typeof value !== "string" || value.trim().length === 0) { @@ -213,6 +219,7 @@ interface ExtensionApiHandle { /** Registration counts captured before one extension runs, for failure rollback. */ interface RegistrySnapshot { themes: number; + syntaxLanguages: number; fileLanguages: number; vcsAdapters: number; changesetTransforms: number; @@ -233,6 +240,7 @@ function snapshotRegistry(registry: ExtensionRegistry): RegistrySnapshot { return { themes: registry.themes.length, + syntaxLanguages: registry.syntaxLanguages.length, fileLanguages: registry.fileLanguages.length, vcsAdapters: registry.vcsAdapters.length, changesetTransforms: registry.changesetTransforms.length, @@ -253,6 +261,7 @@ function snapshotRegistry(registry: ExtensionRegistry): RegistrySnapshot { */ function rollbackRegistry(registry: ExtensionRegistry, snapshot: RegistrySnapshot) { registry.themes.length = snapshot.themes; + registry.syntaxLanguages.length = snapshot.syntaxLanguages; registry.fileLanguages.length = snapshot.fileLanguages; registry.vcsAdapters.length = snapshot.vcsAdapters; registry.changesetTransforms.length = snapshot.changesetTransforms; @@ -323,13 +332,23 @@ export function createExtensionApi( assertNonEmptyString(theme?.id, "registerTheme requires a theme with a non-empty id."); registry.themes.push({ extensionId: metadata.id, theme }); }, + registerSyntaxLanguage(language: string, loader: ExtensionSyntaxLanguageLoader) { + assertOpen("registerSyntaxLanguage"); + if (typeof loader !== "function") { + throw new Error("registerSyntaxLanguage requires a loader function."); + } + registry.syntaxLanguages.push({ + extensionId: metadata.id, + language: normalizeSyntaxLanguageId(language, "registerSyntaxLanguage"), + loader, + }); + }, registerFileLanguage(extension: string, language: string) { assertOpen("registerFileLanguage"); - assertNonEmptyString(language, "registerFileLanguage requires a non-empty language."); registry.fileLanguages.push({ extensionId: metadata.id, extension: normalizeFileExtension(extension), - language, + language: normalizeSyntaxLanguageId(language, "registerFileLanguage"), }); }, registerVcsAdapter(adapter: ExtensionVcsAdapter) { diff --git a/src/extensions/types.ts b/src/extensions/types.ts index 648f823da..4615c9d69 100644 --- a/src/extensions/types.ts +++ b/src/extensions/types.ts @@ -11,6 +11,7 @@ import type { ExtensionFileView, ExtensionNotifyType, ExtensionSidebarView, + ExtensionSyntaxLanguageLoader, ExtensionThemeConfig, } from "../extension-api/types"; import { createExtensionNotificationHub, type ExtensionNotificationHub } from "./notifications"; @@ -65,6 +66,8 @@ export type { ExtensionSidebarTheme, ExtensionSidebarView, ExtensionSidebarViewProps, + ExtensionSyntaxGrammar, + ExtensionSyntaxLanguageLoader, ExtensionThemeConfig, ExtensionVcsAdapter, ExtensionWorkspace, @@ -107,6 +110,12 @@ export interface RegisteredTheme { theme: ExtensionThemeConfig; } +export interface RegisteredSyntaxLanguage { + extensionId: string; + language: string; + loader: ExtensionSyntaxLanguageLoader; +} + export interface RegisteredFileLanguage { extensionId: string; /** Normalized extension without a leading dot, lowercased. */ @@ -173,6 +182,7 @@ export type ExtensionEventHandlerMap = { export interface ExtensionRegistry { extensions: ExtensionMetadata[]; themes: RegisteredTheme[]; + syntaxLanguages: RegisteredSyntaxLanguage[]; fileLanguages: RegisteredFileLanguage[]; vcsAdapters: RegisteredVcsAdapter[]; changesetTransforms: RegisteredChangesetTransform[]; @@ -247,6 +257,7 @@ export function createEmptyExtensionRegistry(): ExtensionRegistry { return { extensions: [], themes: [], + syntaxLanguages: [], fileLanguages: [], vcsAdapters: [], changesetTransforms: [], diff --git a/src/ui/diff/pierre.ts b/src/ui/diff/pierre.ts index 400b70ad6..c2cfea5cf 100644 --- a/src/ui/diff/pierre.ts +++ b/src/ui/diff/pierre.ts @@ -1,12 +1,16 @@ import { cleanLastNewline, + disposeHighlighter, getHighlighterOptions, getSharedHighlighter, renderDiffWithHighlighter, renderFileWithHighlighter, + ResolvingLanguages, + ResolvingThemes, type FileContents, type FileDiffMetadata, } from "@pierre/diffs"; +import { reportSyntaxLanguageFailure } from "../../core/fileLanguage"; import { formatHunkHeader } from "../../core/hunkHeader"; import { DEFAULT_TAB_WIDTH } from "../../core/tabWidth"; import type { DiffFile, DiffLineMoveKind } from "../../core/types"; @@ -519,19 +523,15 @@ async function prepareHighlighter(language: string | undefined, theme: Highlight }); } -/** Queue highlight rendering so startup work stays serialized without starving input/render timers. */ -function queueHighlightedWork(run: () => T) { +/** Queue one complete highlighter lifecycle without starving input/render timers. */ +function queueHighlightedWork(run: () => T | Promise) { const queued = queuedHighlightWork.then( () => new Promise((resolve, reject) => { // Highlighting is CPU-heavy background work. Scheduling each serialized job as a timer, // rather than a microtask, yields back to OpenTUI input and frame timers between files. setTimeout(() => { - try { - resolve(run()); - } catch (error) { - reject(error); - } + Promise.resolve().then(run).then(resolve, reject); }, 0); }), ); @@ -544,6 +544,12 @@ function queueHighlightedWork(run: () => T) { return queued; } +/** Let Pierre's sibling resolver work finish before clearing its shared highlighter state. */ +async function resetHighlighterAfterFailure() { + await Promise.allSettled([...ResolvingLanguages.values(), ...ResolvingThemes.values()]); + await disposeHighlighter(); +} + /** Normalize source text the same way expanded-row slicing does before highlighting. */ function normalizeSourceText(text: string) { return text.replaceAll("\r\n", "\n"); @@ -647,14 +653,8 @@ function renderHighlightedDiff( theme: HighlightThemeInput, sourcePlan: SourceBackedHighlightPlan | null, ) { - return queueHighlightedWork(() => { - const highlighted = renderDiffWithHighlighter( - metadata, - highlighter, - pierreRenderOptions(theme), - ); - return finalizeHighlightedDiff(file, sourcePlan, highlighted); - }); + const highlighted = renderDiffWithHighlighter(metadata, highlighter, pierreRenderOptions(theme)); + return finalizeHighlightedDiff(file, sourcePlan, highlighted); } /** Highlight a diff file and return just the rendered line trees the UI needs. */ @@ -664,36 +664,43 @@ export async function loadHighlightedDiff( ): Promise { const sourcePlan = await loadSourceBackedHighlightPlan(file); - try { - const highlighter = await prepareHighlighter(file.language, theme); + return queueHighlightedWork(async () => { try { - return await renderHighlightedDiff( + const highlighter = await prepareHighlighter(file.language, theme); + try { + return renderHighlightedDiff( + file, + sourcePlan?.metadata ?? file.metadata, + highlighter, + theme, + sourcePlan, + ); + } catch (error) { + if (!sourcePlan) { + throw error; + } + + // A validated source graft should render like ordinary complete-file metadata. If Pierre + // still rejects it, preserve the pre-existing patch-fragment behavior rather than blanking it. + return renderHighlightedDiff(file, file.metadata, highlighter, theme, null); + } + } catch (error) { + reportSyntaxLanguageFailure(file.language, error); + // A rejected grammar can leave Shiki's shared instance partially mutated. + // Wait for sibling theme/language resolvers before cleanup so a late + // attachment cannot mark the replacement highlighter's state as loaded. + await resetHighlighterAfterFailure(); + const fallbackTheme = highlightThemeAppearance(theme); + const highlighter = await prepareHighlighter("text", fallbackTheme); + return renderHighlightedDiff( file, - sourcePlan?.metadata ?? file.metadata, + { ...file.metadata, lang: "text" }, highlighter, - theme, - sourcePlan, + fallbackTheme, + null, ); - } catch (error) { - if (!sourcePlan) { - throw error; - } - - // A validated source graft should render like ordinary complete-file metadata. If Pierre - // still rejects it, preserve the pre-existing patch-fragment behavior rather than blanking it. - return await renderHighlightedDiff(file, file.metadata, highlighter, theme, null); } - } catch { - const fallbackTheme = highlightThemeAppearance(theme); - const highlighter = await prepareHighlighter("text", fallbackTheme); - return await renderHighlightedDiff( - file, - { ...file.metadata, lang: "text" }, - highlighter, - fallbackTheme, - null, - ); - } + }); } /** Highlight a full source file for unchanged lines synthesized during gap expansion. */ @@ -706,9 +713,9 @@ export async function loadHighlightedSourceLines({ text: string; theme?: HighlightThemeInput; }): Promise { - try { - const highlighter = await prepareHighlighter(file.language, theme); - return queueHighlightedWork(() => { + return queueHighlightedWork(async () => { + try { + const highlighter = await prepareHighlighter(file.language, theme); const highlighted = renderFileWithHighlighter( sourceFileContents(file, text, file.language), highlighter, @@ -717,11 +724,11 @@ export async function loadHighlightedSourceLines({ return { lines: highlighted.code as Array, }; - }); - } catch { - const fallbackTheme = highlightThemeAppearance(theme); - const highlighter = await prepareHighlighter("text", fallbackTheme); - return queueHighlightedWork(() => { + } catch (error) { + reportSyntaxLanguageFailure(file.language, error); + await resetHighlighterAfterFailure(); + const fallbackTheme = highlightThemeAppearance(theme); + const highlighter = await prepareHighlighter("text", fallbackTheme); const highlighted = renderFileWithHighlighter( sourceFileContents(file, text, "text"), highlighter, @@ -730,8 +737,8 @@ export async function loadHighlightedSourceLines({ return { lines: highlighted.code as Array, }; - }); - } + } + }); } /** Convert one highlighted full-source line into the spans used by expanded context rows. */ diff --git a/src/ui/diff/useHighlightedDiff.test.ts b/src/ui/diff/useHighlightedDiff.test.ts index b690a7efc..67f98a32c 100644 --- a/src/ui/diff/useHighlightedDiff.test.ts +++ b/src/ui/diff/useHighlightedDiff.test.ts @@ -4,6 +4,15 @@ import { resolveTheme } from "../themes"; import { highlightedDiffCacheKey } from "./useHighlightedDiff"; describe("highlighted diff cache keys", () => { + test("invalidates a cached result when extension reload changes the detected language", () => { + const file = createTestDiffFile({ id: "cache-language", path: "example.custom" }); + const theme = resolveTheme("github-dark-default", null); + + expect(highlightedDiffCacheKey(theme, { ...file, language: undefined })).not.toBe( + highlightedDiffCacheKey(theme, { ...file, language: "custom-syntax" }), + ); + }); + test("invalidates source-backed partial highlights when an unversioned provider changes", () => { const base = createTestDiffFile({ id: "cache", path: "cache.ts" }); const firstFetcher = createTestSourceFetcher(() => "first source\n"); diff --git a/src/ui/diff/useHighlightedDiff.ts b/src/ui/diff/useHighlightedDiff.ts index 5a6da0b76..71a481bc8 100644 --- a/src/ui/diff/useHighlightedDiff.ts +++ b/src/ui/diff/useHighlightedDiff.ts @@ -102,7 +102,7 @@ function sourceFetcherFingerprint(file: DiffFile) { /** Cache key that includes patch and source-provider identity so reloads cannot reuse stale grammar state. */ export function highlightedDiffCacheKey(theme: AppTheme, file: DiffFile) { - return `${theme.id}:${syntaxHighlightThemeName(theme)}:${file.id}:${patchFingerprint(file)}:${sourceFetcherFingerprint(file)}`; + return `${theme.id}:${syntaxHighlightThemeName(theme)}:${file.language ?? "text"}:${file.id}:${patchFingerprint(file)}:${sourceFetcherFingerprint(file)}`; } /** Only commit a highlight result if the promise is still the active one for that key. diff --git a/website/src/content/docs/docs/extend/extension-api.md b/website/src/content/docs/docs/extend/extension-api.md index fac90e92b..1d0dc5d3e 100644 --- a/website/src/content/docs/docs/extend/extension-api.md +++ b/website/src/content/docs/docs/extend/extension-api.md @@ -1,13 +1,13 @@ --- title: Extension API -description: Register themes, file previews, transforms, commands, dialogs, and events through the extension API object. +description: Register themes, syntax languages, file previews, transforms, commands, dialogs, and events through the extension API object. --- The extension factory receives one API object. Registration calls are only valid while the factory is running; Hunk seals the object afterwards so a deferred callback cannot mutate the registry mid-session. This page indexes the whole object; larger registration calls are documented in depth on their own pages and summarized in place below. ## `hunk.apiVersion` -The API generation this Hunk speaks (currently `2`). Branch on it if you want one file to support several Hunk versions. Version 2 adds the experimental file-view contract and its command controls. +The API generation this Hunk speaks (currently `3`). Branch on it if you want one file to support several Hunk versions. Version 3 adds lazy custom syntax-language registration. ## `hunk.registerTheme(theme)` @@ -25,9 +25,29 @@ hunk.registerTheme({ Theme ids are lowercase words separated by `-` or `_` and cannot reuse a built-in id. Config-defined themes win over extension themes for the same id. Extension themes appear in the selector after config themes, in load order. +## `hunk.registerSyntaxLanguage(language, loader)` + +Register a lazy Shiki-compatible TextMate grammar under a highlighting language id. The loader resolves to an ES-module-shaped object with a non-empty grammar array as its `default` export: + +```ts +hunk.registerSyntaxLanguage("example-lang", async () => ({ + default: [ + { + name: "example-lang", + scopeName: "source.example-lang", + patterns: [{ match: "\\bexample\\b", name: "keyword.control.example-lang" }], + repository: {}, + }, + ], +})); +hunk.registerFileLanguage("example", "example-lang"); +``` + +Folder extensions can install a grammar package and pass its dynamic import directly, such as `() => import("@shikijs/langs/odin")`. Syntax registrations last for the Hunk process; restart Hunk to replace or remove one. The first custom registration for an id wins, while `text` and `ansi` remain reserved. + ## `hunk.registerFileLanguage(extension, language)` -Map a file extension to a syntax-highlighting language. The extension may be written with or without a leading dot and is lowercased. +Map a file extension to a syntax-highlighting language. The extension may be written with or without a leading dot and is lowercased. Register a custom syntax language first when Pierre does not bundle it. ```ts hunk.registerFileLanguage(".zig", "zig"); From 3e791a05fff9541ebf33a127737c9e9db8b98b8c Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sat, 8 Aug 2026 02:41:06 -0400 Subject: [PATCH 2/5] fix(extensions): isolate syntax loader failures --- docs/extensions.md | 8 +- src/extensions/apply.test.ts | 119 ++++++++++++++++++ src/extensions/apply.ts | 79 ++++++++---- src/ui/diff/pierre.ts | 27 ++-- src/ui/diff/useHighlightedDiff.test.ts | 35 +++++- src/ui/diff/useHighlightedDiff.ts | 14 ++- .../content/docs/docs/extend/extension-api.md | 2 +- 7 files changed, 239 insertions(+), 45 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index 68283fed5..9f122d2a5 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -249,9 +249,11 @@ so choose a new id unless replacing the built-in grammar is deliberate. Pierre's grammar registry lasts for the Hunk process and cannot safely replace or unregister a loader. Reloading the same extension is idempotent, but grammar changes, removals, and replacing a grammar already used by this process require -restarting Hunk. A loader that rejects, returns an invalid module, or supplies a -grammar Shiki cannot attach produces one attributed warning when first used, -and that file falls back to unhighlighted text. +restarting Hunk. A loader that rejects, takes longer than five seconds, returns +an invalid module, or supplies a grammar Shiki cannot attach produces one +attributed warning when first used, and that file falls back to unhighlighted +text. Fallback results are retried when the file is highlighted again, so a +transient loader failure does not remain cached for the session. ### `hunk.registerFileLanguage(extension, language)` diff --git a/src/extensions/apply.test.ts b/src/extensions/apply.test.ts index 9d07adb17..55f38ae33 100644 --- a/src/extensions/apply.test.ts +++ b/src/extensions/apply.test.ts @@ -13,6 +13,7 @@ import { getFiletypeFromFileName } from "../core/fileLanguage"; import type { Changeset, DiffFile } from "../core/types"; import type { VcsAdapter } from "../core/vcs/types"; import { buildStackRows, loadHighlightedDiff } from "../ui/diff/pierre"; +import { prefetchHighlightedDiff } from "../ui/diff/useHighlightedDiff"; import { resolveTheme } from "../ui/themes"; import { applyExtensionChangesetTransforms, @@ -252,6 +253,83 @@ describe("extension syntax languages", () => { ); }); + test("times out a stalled loader without blocking later highlighting", async () => { + const { result, notices } = createTestLoadResult(); + const language = "hunk-stalled-syntax-fixture"; + result.registry.syntaxLanguages.push({ + extensionId: "stalled-syntax", + language, + loader: () => new Promise(() => undefined), + }); + applyExtensionSyntaxLanguages(result.registry, result.context, { loaderTimeoutMs: 10 }); + + const stalledFile = createTestDiffFile({ + id: "stalled-syntax", + language, + path: "stalled.hunksyntaxfixture", + }); + const recoveryFile = createTestDiffFile({ + id: "stalled-syntax-recovery", + language: "typescript", + path: "recovery.ts", + }); + const theme = resolveTheme("github-dark-default", null); + const [fallback, recovery] = await Promise.all([ + loadHighlightedDiff(stalledFile, theme), + loadHighlightedDiff(recoveryFile, theme), + ]); + + expect(fallback.cachePolicy).toBe("retry"); + expect(recovery.cachePolicy).toBe("reuse"); + expect(notices).toHaveLength(1); + expect(notices[0]).toContain("loader did not resolve within 10ms"); + const recoverySpans = buildStackRows(recoveryFile, recovery, theme) + .filter((row) => row.type === "stack-line" && row.cell.kind === "addition") + .flatMap((row) => (row.type === "stack-line" ? row.cell.spans : [])); + expect(recoverySpans.some((span) => span.fg !== undefined)).toBe(true); + }); + + test("retries plaintext fallbacks instead of retaining them in the shared cache", async () => { + const { result } = createTestLoadResult(); + const language = "hunk-transient-syntax-fixture"; + let loadAttempts = 0; + result.registry.syntaxLanguages.push({ + extensionId: "transient-syntax", + language, + loader: async () => { + loadAttempts += 1; + if (loadAttempts === 1) { + throw new Error("transient grammar failure"); + } + return createTestSyntaxLoader(language)(); + }, + }); + applyExtensionSyntaxLanguages(result.registry, result.context); + + const theme = resolveTheme("github-dark-default", null); + const firstFile = createTestDiffFile({ + id: "transient-syntax-first", + language, + path: "first.hunksyntaxfixture", + }); + const secondFile = createTestDiffFile({ + id: "transient-syntax-second", + language, + path: "second.hunksyntaxfixture", + }); + + const fallback = await prefetchHighlightedDiff({ file: firstFile, theme }); + const recovered = await prefetchHighlightedDiff({ file: secondFile, theme }); + const revisited = await prefetchHighlightedDiff({ file: firstFile, theme }); + + expect(fallback.cachePolicy).toBe("retry"); + expect(recovered.cachePolicy).toBe("reuse"); + expect(revisited.cachePolicy).toBe("reuse"); + expect(revisited).not.toBe(fallback); + expect(await prefetchHighlightedDiff({ file: firstFile, theme })).toBe(revisited); + expect(loadAttempts).toBe(2); + }); + test("attributes grammar attachment failures before falling back to text", async () => { const { result, notices } = createTestLoadResult(); const language = "hunk-broken-render-syntax-fixture"; @@ -371,6 +449,47 @@ describe("extension syntax languages", () => { }, ]); }); + + test("does not let a reload challenger claim the retained owner's language", () => { + const language = "hunk-reload-order-syntax-fixture"; + const initial = createTestLoadResult().result; + initial.registry.extensions.push({ + id: "owner", + sourcePath: "/extensions/owner.ts", + origin: "global", + }); + initial.registry.syntaxLanguages.push({ + extensionId: "owner", + language, + loader: createTestSyntaxLoader(language), + }); + expect(applyExtensionSyntaxLanguages(initial.registry, initial.context)).toEqual([]); + + const reload = createTestLoadResult().result; + reload.registry.extensions.push( + { id: "challenger", sourcePath: "/extensions/challenger.ts", origin: "global" }, + { id: "owner", sourcePath: "/extensions/owner.ts", origin: "global" }, + ); + reload.registry.syntaxLanguages.push( + { + extensionId: "challenger", + language, + loader: createTestSyntaxLoader("ignored-reload-order-syntax-fixture"), + }, + { + extensionId: "owner", + language, + loader: createTestSyntaxLoader(language), + }, + ); + + expect(applyExtensionSyntaxLanguages(reload.registry, reload.context)).toEqual([ + { + extensionId: "challenger", + message: `Skipped syntax language "${language}" from extension challenger • already loaded by extension owner; restart Hunk to replace it`, + }, + ]); + }); }); describe("extension file languages", () => { diff --git a/src/extensions/apply.ts b/src/extensions/apply.ts index b96d47601..74e7cdd2a 100644 --- a/src/extensions/apply.ts +++ b/src/extensions/apply.ts @@ -5,6 +5,7 @@ import { } from "../core/fileLanguage"; import type { StartupNotice } from "../core/startupNotice"; import type { Changeset } from "../core/types"; +import type { ExtensionSyntaxGrammar, ExtensionSyntaxLanguageLoader } from "../extension-api/types"; import { detectVcs, getDefaultVcsAdapter, isVcsId, resolveVcsAdapters } from "../core/vcs"; import type { VcsAdapter } from "../core/vcs/types"; import { sanitizeTerminalLine } from "../lib/terminalText"; @@ -39,6 +40,7 @@ function describeError(error: unknown) { } const RESERVED_SYNTAX_LANGUAGE_IDS = new Set(["text", "ansi"]); +export const EXTENSION_SYNTAX_LANGUAGE_LOAD_TIMEOUT_MS = 5_000; interface AppliedSyntaxLanguage { extensionId: string; @@ -49,32 +51,66 @@ interface AppliedSyntaxLanguage { // the process and extension reloads may only add previously unseen language ids. const appliedSyntaxLanguages = new Map(); +type SyntaxLanguageModule = Awaited>; + +interface ApplyExtensionSyntaxLanguageOptions { + loaderTimeoutMs?: number; +} + +/** Report whether a value is a non-empty string. */ +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.trim().length > 0; +} + +/** Report whether a value has the required TextMate grammar fields. */ +function isSyntaxGrammar(value: unknown): value is ExtensionSyntaxGrammar { + if (typeof value !== "object" || value === null) { + return false; + } + + const grammar = value as { name?: unknown; scopeName?: unknown }; + return isNonEmptyString(grammar.name) && isNonEmptyString(grammar.scopeName); +} + /** Report whether one lazy module contains usable TextMate grammar records. */ -function isSyntaxLanguageModule( - value: unknown, -): value is Awaited> { - const module = value as { default?: unknown } | null; - return ( - typeof module === "object" && - module !== null && - Array.isArray(module.default) && - module.default.length > 0 && - module.default.every( - (grammar) => - typeof grammar === "object" && - grammar !== null && - typeof (grammar as { name?: unknown }).name === "string" && - (grammar as { name: string }).name.trim().length > 0 && - typeof (grammar as { scopeName?: unknown }).scopeName === "string" && - (grammar as { scopeName: string }).scopeName.trim().length > 0, - ) - ); +function isSyntaxLanguageModule(value: unknown): value is SyntaxLanguageModule { + if (typeof value !== "object" || value === null) { + return false; + } + + const grammars = (value as { default?: unknown }).default; + return Array.isArray(grammars) && grammars.length > 0 && grammars.every(isSyntaxGrammar); +} + +/** Bound extension-controlled grammar loading so one loader cannot block all highlighting. */ +async function loadSyntaxLanguageWithin( + loader: ExtensionSyntaxLanguageLoader, + timeoutMs: number, +): Promise { + const loading = Promise.resolve().then(loader); + let timeout: ReturnType; + const deadline = new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error(`loader did not resolve within ${timeoutMs}ms`)), + timeoutMs, + ); + }); + + try { + return await Promise.race([loading, deadline]); + } finally { + clearTimeout(timeout!); + loading.catch(() => undefined); + } } /** Add extension-contributed lazy grammars to Pierre's process-wide registry. */ export function applyExtensionSyntaxLanguages( registry: ExtensionRegistry, context?: ExtensionContext, + { + loaderTimeoutMs = EXTENSION_SYNTAX_LANGUAGE_LOAD_TIMEOUT_MS, + }: ApplyExtensionSyntaxLanguageOptions = {}, ): ExtensionApplyIssue[] { const issues: ExtensionApplyIssue[] = []; const claimedThisPass = new Map(); @@ -96,7 +132,6 @@ export function applyExtensionSyntaxLanguages( }); continue; } - claimedThisPass.set(language, extensionId); const sourcePath = registry.extensions.find((extension) => extension.id === extensionId)?.sourcePath ?? @@ -106,6 +141,7 @@ export function applyExtensionSyntaxLanguages( // Reapplying the same extension is idempotent. Pierre cannot replace the // old loader, so a changed grammar takes effect after Hunk restarts. if (applied.extensionId === extensionId && applied.sourcePath === sourcePath) { + claimedThisPass.set(language, extensionId); continue; } issues.push({ @@ -128,7 +164,7 @@ export function applyExtensionSyntaxLanguages( }; const attributedLoader: typeof loader = async () => { try { - const module = await loader(); + const module = await loadSyntaxLanguageWithin(loader, loaderTimeoutMs); if (!isSyntaxLanguageModule(module)) { throw new Error( "loader must resolve to { default: Grammar[] } with non-empty name and scopeName fields", @@ -144,6 +180,7 @@ export function applyExtensionSyntaxLanguages( try { registerSyntaxLanguage(language, attributedLoader, reportFailure); appliedSyntaxLanguages.set(language, { extensionId, sourcePath }); + claimedThisPass.set(language, extensionId); } catch (error) { issues.push({ extensionId, diff --git a/src/ui/diff/pierre.ts b/src/ui/diff/pierre.ts index c2cfea5cf..85b99dea5 100644 --- a/src/ui/diff/pierre.ts +++ b/src/ui/diff/pierre.ts @@ -69,6 +69,7 @@ interface HastElementNode { export interface HighlightedDiffCode { deletionLines: Array; additionLines: Array; + cachePolicy: "reuse" | "retry"; } export interface HighlightedSourceCode { @@ -636,13 +637,16 @@ function finalizeHighlightedDiff( const code = { deletionLines: highlighted.code.deletionLines as Array, additionLines: highlighted.code.additionLines as Array, + cachePolicy: "reuse" as const, }; // Full old/new sources can put identical context text in different lexical states. Preserve // those authoritative per-side nodes; aliasing remains safe only for patch-fragment highlighting. - return sourcePlan - ? remapSourceBackedHighlight(sourcePlan, code) - : aliasHighlightedContextLines(file, code); + if (sourcePlan) { + return { ...remapSourceBackedHighlight(sourcePlan, code), cachePolicy: "reuse" }; + } + + return aliasHighlightedContextLines(file, code); } /** Render one metadata snapshot through an already prepared highlighter. */ @@ -692,13 +696,16 @@ export async function loadHighlightedDiff( await resetHighlighterAfterFailure(); const fallbackTheme = highlightThemeAppearance(theme); const highlighter = await prepareHighlighter("text", fallbackTheme); - return renderHighlightedDiff( - file, - { ...file.metadata, lang: "text" }, - highlighter, - fallbackTheme, - null, - ); + return { + ...renderHighlightedDiff( + file, + { ...file.metadata, lang: "text" }, + highlighter, + fallbackTheme, + null, + ), + cachePolicy: "retry", + }; } }); } diff --git a/src/ui/diff/useHighlightedDiff.test.ts b/src/ui/diff/useHighlightedDiff.test.ts index 67f98a32c..fc89fc9e8 100644 --- a/src/ui/diff/useHighlightedDiff.test.ts +++ b/src/ui/diff/useHighlightedDiff.test.ts @@ -1,16 +1,41 @@ import { describe, expect, test } from "bun:test"; import { createTestDiffFile, createTestSourceFetcher } from "../../../test/helpers/diff-helpers"; import { resolveTheme } from "../themes"; -import { highlightedDiffCacheKey } from "./useHighlightedDiff"; +import { buildStackRows } from "./pierre"; +import { highlightedDiffCacheKey, prefetchHighlightedDiff } from "./useHighlightedDiff"; describe("highlighted diff cache keys", () => { - test("invalidates a cached result when extension reload changes the detected language", () => { - const file = createTestDiffFile({ id: "cache-language", path: "example.custom" }); + test("loads a new cached result when extension reload changes the detected language", async () => { + const baseFile = createTestDiffFile({ id: "cache-language", path: "example.custom" }); + const plainFile = { + ...baseFile, + language: undefined, + metadata: { ...baseFile.metadata, lang: "text" as const }, + }; + const typedFile = { + ...baseFile, + language: "typescript", + metadata: { ...baseFile.metadata, lang: "typescript" as const }, + }; const theme = resolveTheme("github-dark-default", null); - expect(highlightedDiffCacheKey(theme, { ...file, language: undefined })).not.toBe( - highlightedDiffCacheKey(theme, { ...file, language: "custom-syntax" }), + const plain = await prefetchHighlightedDiff({ file: plainFile, theme }); + const typed = await prefetchHighlightedDiff({ file: typedFile, theme }); + const typedAgain = await prefetchHighlightedDiff({ file: typedFile, theme }); + const plainSpans = buildStackRows(plainFile, plain, theme) + .filter((row) => row.type === "stack-line" && row.cell.kind === "addition") + .flatMap((row) => (row.type === "stack-line" ? row.cell.spans : [])); + const typedSpans = buildStackRows(typedFile, typed, theme) + .filter((row) => row.type === "stack-line" && row.cell.kind === "addition") + .flatMap((row) => (row.type === "stack-line" ? row.cell.spans : [])); + + expect(highlightedDiffCacheKey(theme, plainFile)).not.toBe( + highlightedDiffCacheKey(theme, typedFile), ); + expect(typed).not.toBe(plain); + expect(typedAgain).toBe(typed); + expect(plainSpans.some((span) => span.fg !== undefined)).toBe(false); + expect(typedSpans.some((span) => span.fg !== undefined)).toBe(true); }); test("invalidates source-backed partial highlights when an unversioned provider changes", () => { diff --git a/src/ui/diff/useHighlightedDiff.ts b/src/ui/diff/useHighlightedDiff.ts index 71a481bc8..5ea9afd19 100644 --- a/src/ui/diff/useHighlightedDiff.ts +++ b/src/ui/diff/useHighlightedDiff.ts @@ -105,9 +105,8 @@ export function highlightedDiffCacheKey(theme: AppTheme, file: DiffFile) { return `${theme.id}:${syntaxHighlightThemeName(theme)}:${file.language ?? "text"}:${file.id}:${patchFingerprint(file)}:${sourceFetcherFingerprint(file)}`; } -/** Only commit a highlight result if the promise is still the active one for that key. - * Prevents a superseded or late-resolving promise from overwriting a newer entry. */ -function commitHighlightResult( +/** Settle the active request, caching only results that do not need a later retry. */ +function settleHighlightResult( cacheKey: string, promise: Promise, result: HighlightedDiffCode, @@ -117,6 +116,10 @@ function commitHighlightResult( } SHARED_HIGHLIGHT_PROMISES.delete(cacheKey); + if (result.cachePolicy === "retry") { + return true; + } + SHARED_HIGHLIGHTED_DIFF_CACHE.set(cacheKey, result); enforceCacheLimit(); return true; @@ -141,15 +144,16 @@ function ensureHighlightedDiffLoaded( let pending: Promise; pending = loadHighlightedDiff(file, theme) .then((nextHighlighted) => { - commitHighlightResult(cacheKey, pending, nextHighlighted); + settleHighlightResult(cacheKey, pending, nextHighlighted); return nextHighlighted; }) .catch(() => { const fallback = { deletionLines: [], additionLines: [], + cachePolicy: "retry", } satisfies HighlightedDiffCode; - commitHighlightResult(cacheKey, pending, fallback); + settleHighlightResult(cacheKey, pending, fallback); return fallback; }); diff --git a/website/src/content/docs/docs/extend/extension-api.md b/website/src/content/docs/docs/extend/extension-api.md index 1d0dc5d3e..48e56d32b 100644 --- a/website/src/content/docs/docs/extend/extension-api.md +++ b/website/src/content/docs/docs/extend/extension-api.md @@ -43,7 +43,7 @@ hunk.registerSyntaxLanguage("example-lang", async () => ({ hunk.registerFileLanguage("example", "example-lang"); ``` -Folder extensions can install a grammar package and pass its dynamic import directly, such as `() => import("@shikijs/langs/odin")`. Syntax registrations last for the Hunk process; restart Hunk to replace or remove one. The first custom registration for an id wins, while `text` and `ansi` remain reserved. +Folder extensions can install a grammar package and pass its dynamic import directly, such as `() => import("@shikijs/langs/odin")`. Syntax registrations last for the Hunk process; restart Hunk to replace or remove one. The first custom registration for an id wins, while `text` and `ansi` remain reserved. Loaders that reject, take longer than five seconds, or return an invalid grammar fall back to plaintext with an attributed warning. Hunk retries that fallback when the file is highlighted again. ## `hunk.registerFileLanguage(extension, language)` From a5187a2fc5a114a3766a1d406dedbffb271daa18 Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sat, 8 Aug 2026 08:48:20 -0400 Subject: [PATCH 3/5] test(windows): retry locked fixture cleanup --- src/ui/AppHost.workspace.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ui/AppHost.workspace.test.tsx b/src/ui/AppHost.workspace.test.tsx index 0a58e3001..51515711e 100644 --- a/src/ui/AppHost.workspace.test.tsx +++ b/src/ui/AppHost.workspace.test.tsx @@ -31,7 +31,8 @@ const tempDirs: string[] = []; afterEach(() => { for (const dir of tempDirs.splice(0)) { - rmSync(dir, { recursive: true, force: true }); + // Windows keeps a Git fixture locked briefly while background source highlighting exits. + rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } }); From 3113bfdde21db8e254d6f4eee1ff0005320fd12e Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sat, 8 Aug 2026 09:52:41 -0400 Subject: [PATCH 4/5] fix(extensions): resolve dependencies in compiled builds --- bun.lock | 9 + docs/extension-architecture.md | 13 +- package.json | 3 + src/extensions/hostRuntimeModules.test.ts | 135 +++++++++++- src/extensions/hostRuntimeModules.ts | 254 ++++++++++++++++++++-- 5 files changed, 389 insertions(+), 25 deletions(-) diff --git a/bun.lock b/bun.lock index c96754468..52e11f4ae 100644 --- a/bun.lock +++ b/bun.lock @@ -6,11 +6,14 @@ "name": "hunk", "dependencies": { "@pierre/diffs": "1.2.2", + "acorn": "8.15.0", "bun": "^1.3.14", "chokidar": "^4.0.3", "commander": "^14.0.3", "diff": "^8.0.3", + "es-module-lexer": "1.7.0", "get-east-asian-width": "^1.5.0", + "import-meta-resolve": "4.2.0", "shell-quote": "1.9.0", "string-width": "^8.2.1", "zod": "^4.3.6", @@ -275,6 +278,8 @@ "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="], + "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], + "ansi-escapes": ["ansi-escapes@7.3.0", "", { "dependencies": { "environment": "^1.0.0" } }, "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg=="], "ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], @@ -329,6 +334,8 @@ "errore": ["errore@0.11.0", "", { "bin": { "errore": "dist/cli.js" } }, "sha512-/uJh8o4SYfJAPGSDynpLgKRuRWX5yTSP2BXspHVQu8XmwaX1d6ysxr1cBhjTzC1Um2Xov9BQJ2kigT9lvxHYaA=="], + "es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="], + "eventemitter3": ["eventemitter3@5.0.4", "", {}, "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw=="], "execa": ["execa@0.9.0", "", { "dependencies": { "cross-spawn": "^5.0.1", "get-stream": "^3.0.0", "is-stream": "^1.1.0", "npm-run-path": "^2.0.0", "p-finally": "^1.0.0", "signal-exit": "^3.0.0", "strip-eof": "^1.0.0" } }, "sha512-BbUMBiX4hqiHZUA5+JujIjNb6TyAlp2D5KLheMjMluwOuzcnylDL4AxZYLLn1n2AGB49eSWwyKvvEQoRpnAtmA=="], @@ -351,6 +358,8 @@ "html-void-elements": ["html-void-elements@3.0.0", "", {}, "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg=="], + "import-meta-resolve": ["import-meta-resolve@4.2.0", "", {}, "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg=="], + "is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], "is-stream": ["is-stream@1.1.0", "", {}, "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ=="], diff --git a/docs/extension-architecture.md b/docs/extension-architecture.md index 2ae542f5d..12b39ca95 100644 --- a/docs/extension-architecture.md +++ b/docs/extension-architecture.md @@ -64,10 +64,15 @@ host-served runtime modules (`src/extensions/hostRuntimeModules.ts`): a per-extension-directory Bun loader hook transpiles extension source and rewrites those specifiers to prefixed virtual modules backed by the host's own instances. That identity is what lets `registerSidebarView` components -render inside the app's React tree with working hooks. The module header -documents why the obvious alternatives don't work (process-wide specifier -claims break the host's lazy imports; the loaders resolve lazily so headless -commands never pay OpenTUI's native-library extraction). +render inside the app's React tree with working hooks. Other literal imports +resolve from the extension's own `node_modules` and are rewritten to filesystem +URLs, including package export/import maps, because Bun's compiled runtime +cannot resolve packages that were installed after Hunk itself was built. Resolved +package modules join the same scoped loader path so their dependencies work +recursively. The module header documents why the obvious alternatives don't +work (process-wide specifier claims break the host's lazy imports; the loaders +resolve lazily so headless commands never pay OpenTUI's native-library +extraction). ## Sidebar system diff --git a/package.json b/package.json index f03d0e69d..5c4ecca25 100644 --- a/package.json +++ b/package.json @@ -109,11 +109,14 @@ }, "dependencies": { "@pierre/diffs": "1.2.2", + "acorn": "8.15.0", "bun": "^1.3.14", "chokidar": "^4.0.3", "commander": "^14.0.3", "diff": "^8.0.3", + "es-module-lexer": "1.7.0", "get-east-asian-width": "^1.5.0", + "import-meta-resolve": "4.2.0", "shell-quote": "1.9.0", "string-width": "^8.2.1", "zod": "^4.3.6" diff --git a/src/extensions/hostRuntimeModules.test.ts b/src/extensions/hostRuntimeModules.test.ts index eba523dd6..b4d1f5a20 100644 --- a/src/extensions/hostRuntimeModules.test.ts +++ b/src/extensions/hostRuntimeModules.test.ts @@ -6,7 +6,11 @@ import { afterEach, describe, expect, test } from "bun:test"; import { TextAttributes } from "@opentui/core"; import { isValidElement, useState } from "react"; import { HunkExtensionUserError } from "../extension-api"; -import { registerHostRuntimeModules } from "./hostRuntimeModules"; +import { + registerHostRuntimeModules, + rewriteExtensionDependencySpecifiers, + rewriteHostSpecifiers, +} from "./hostRuntimeModules"; /** * These tests import real files from a temp directory, the way extension @@ -146,6 +150,135 @@ describe("registerHostRuntimeModules", () => { expect(mod.default.helperUseState).toBe(useState); }); + test("resolves package exports when compiled Bun cannot resolve filesystem packages", () => { + const path = writeTempExtension( + "ext.ts", + `export default () => import("@fixture/langs/odin");\n`, + ); + const packageDir = join(dirname(path), "node_modules", "@fixture", "langs"); + const grammarPath = join(packageDir, "dist", "odin.mjs"); + const internalPath = join(packageDir, "dist", "grammar.mjs"); + mkdirSync(dirname(grammarPath), { recursive: true }); + writeFileSync( + join(packageDir, "package.json"), + JSON.stringify({ + name: "@fixture/langs", + type: "module", + exports: { "./odin": "./dist/odin.mjs" }, + imports: { "#grammar": "./dist/grammar.mjs" }, + }), + ); + writeFileSync(grammarPath, `import grammar from "#grammar";\nexport default grammar;\n`); + writeFileSync(internalPath, `export default [{ name: "odin", scopeName: "source.odin" }];\n`); + + const rewritten = rewriteExtensionDependencySpecifiers( + `export default () => import("@fixture/langs/odin");\n`, + path, + () => { + throw new Error("compiled resolver unavailable"); + }, + ); + + const rewrittenInternal = rewriteExtensionDependencySpecifiers( + `import grammar from "#grammar";\nexport default grammar;\n`, + grammarPath, + () => { + throw new Error("compiled resolver unavailable"); + }, + ); + + const importLikeData = + `const quoted = 'import("@fixture/langs/odin")';\n` + + `const templated = \`import("@fixture/langs/odin")\`;\n` + + `// import("@fixture/langs/odin")\n` + + `export default quoted + templated;\n`; + + expect(rewritten).toContain(JSON.stringify(pathToFileURL(grammarPath).href)); + expect(rewrittenInternal).toContain(JSON.stringify(pathToFileURL(internalPath).href)); + expect( + rewriteExtensionDependencySpecifiers(importLikeData, path, () => { + throw new Error("compiled resolver unavailable"); + }), + ).toBe(importLikeData); + expect(rewriteHostSpecifiers(`export default 'import("react")';\n`)).toBe( + `export default 'import("react")';\n`, + ); + }); + + test("matches Bun's main-before-module legacy package fallback", () => { + const path = writeTempExtension("ext.ts", `export { default } from "fixture-legacy";\n`); + const packageDir = join(dirname(path), "node_modules", "fixture-legacy"); + const mainPath = join(packageDir, "main.cjs"); + mkdirSync(packageDir, { recursive: true }); + writeFileSync( + join(packageDir, "package.json"), + JSON.stringify({ + name: "fixture-legacy", + main: "./main.cjs", + module: "./module.mjs", + }), + ); + writeFileSync(mainPath, `module.exports = "main";\n`); + writeFileSync(join(packageDir, "module.mjs"), `export default "module";\n`); + + const rewritten = rewriteExtensionDependencySpecifiers( + `export { default } from "fixture-legacy";\n`, + path, + () => { + throw new Error("compiled resolver unavailable"); + }, + ); + + const regexData = `export default /require("fixture-legacy")/;\n`; + const templateExpression = `export default \`\${require("fixture-legacy")}\`;\n`; + const rewrittenTemplate = rewriteExtensionDependencySpecifiers(templateExpression, path, () => { + throw new Error("compiled resolver unavailable"); + }); + + expect(rewritten).toContain(JSON.stringify(pathToFileURL(mainPath).href)); + expect( + rewriteExtensionDependencySpecifiers(regexData, path, () => { + throw new Error("compiled resolver unavailable"); + }), + ).toBe(regexData); + expect(rewrittenTemplate).toContain(JSON.stringify(mainPath)); + }); + + test("loads a folder extension's lazy package and its package dependencies", async () => { + const path = writeTempExtension( + "ext.ts", + `export default { load: () => import("fixture-parent") };\n`, + ); + const nodeModules = join(dirname(path), "node_modules"); + const parentDir = join(nodeModules, "fixture-parent"); + const childDir = join(nodeModules, "fixture-child"); + mkdirSync(parentDir, { recursive: true }); + mkdirSync(childDir, { recursive: true }); + writeFileSync( + join(parentDir, "package.json"), + JSON.stringify({ + name: "fixture-parent", + type: "module", + exports: "./index.js", + imports: { "#child": "fixture-child" }, + }), + ); + writeFileSync( + join(parentDir, "index.js"), + `import value from "#child";\nexport default { value };\n`, + ); + writeFileSync( + join(childDir, "package.json"), + JSON.stringify({ name: "fixture-child", type: "module", exports: "./index.js" }), + ); + writeFileSync(join(childDir, "index.js"), `export default "from nested dependency";\n`); + + const mod = await importTempExtension(path); + const load = mod.default.load as () => Promise<{ default: { value: string } }>; + + expect((await load()).default.value).toBe("from nested dependency"); + }); + test("does not claim bare specifiers outside registered extension directories", async () => { // The load hook is scoped per directory on purpose: a process-wide claim on // `react` breaks the host's own lazily imported modules when Hunk runs from diff --git a/src/extensions/hostRuntimeModules.ts b/src/extensions/hostRuntimeModules.ts index 981bab943..0f94caff8 100644 --- a/src/extensions/hostRuntimeModules.ts +++ b/src/extensions/hostRuntimeModules.ts @@ -1,4 +1,9 @@ -import { dirname } from "node:path"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parse as parseJavaScript } from "acorn"; +import { parse as parseModuleSpecifiers } from "es-module-lexer/js"; +import { moduleResolve } from "import-meta-resolve"; /** * Host-owned modules served to dynamically imported extension files. @@ -24,6 +29,12 @@ import { dirname } from "node:path"; * `@opentui/react/jsx-runtime` under a pragma), and they only pass through the * rewrite if they exist before Bun's own loader would have added them. * + * The same pass resolves extension-owned imports against adjacent + * `node_modules` and rewrites them to filesystem URLs. Bun's compiled runtime + * cannot resolve packages installed after the executable was built, so Hunk + * performs the package-exports lookup itself and enrolls resolved dependency + * directories in the scoped loader path recursively. + * * Modules linked into the compiled binary never re-resolve their imports, so * none of this affects the host bundle in any run mode. */ @@ -56,28 +67,227 @@ const HOST_MODULE_LOADERS: Record Promise> = { /** Namespace for the virtual modules, chosen to never collide with a real package. */ const HOST_MODULE_PREFIX = "hunk-host:"; +/** Specifier schemes Bun already resolves without consulting extension node_modules. */ +const RUNTIME_SPECIFIER_SCHEME = /^(?:bun|data|file|hunk-host|node):/; + +interface ModuleSpecifierRange { + start: number; + end: number; + specifier: string; + requireCondition: boolean; +} + +interface JavaScriptNode { + type?: string; + start?: number; + end?: number; + name?: string; + value?: unknown; + callee?: JavaScriptNode; + arguments?: JavaScriptNode[]; + [key: string]: unknown; +} + +/** Collect literal CommonJS require calls from a real JavaScript syntax tree. */ +function collectRequireSpecifiers(code: string): ModuleSpecifierRange[] { + let root: JavaScriptNode; + try { + root = parseJavaScript(code, { + ecmaVersion: "latest", + sourceType: "module", + allowAwaitOutsideFunction: true, + allowReturnOutsideFunction: true, + }) as unknown as JavaScriptNode; + } catch { + // ESM imports still resolve through es-module-lexer; preserve unsupported syntax unchanged. + return []; + } + + const ranges: ModuleSpecifierRange[] = []; + const visit = (value: unknown) => { + if (Array.isArray(value)) { + for (const item of value) visit(item); + return; + } + if (typeof value !== "object" || value === null) { + return; + } + + const node = value as JavaScriptNode; + const argument = node.arguments?.[0]; + if ( + node.type === "CallExpression" && + node.callee?.type === "Identifier" && + node.callee.name === "require" && + node.arguments?.length === 1 && + argument?.type === "Literal" && + typeof argument.value === "string" && + typeof argument.start === "number" && + typeof argument.end === "number" + ) { + ranges.push({ + start: argument.start + 1, + end: argument.end - 1, + specifier: argument.value, + requireCondition: true, + }); + } + + for (const [key, child] of Object.entries(node)) { + if (key !== "start" && key !== "end") visit(child); + } + }; + visit(root); + return ranges; +} + +/** Collect syntax-aware static imports, dynamic imports, re-exports, and requires. */ +function collectModuleSpecifiers(code: string): ModuleSpecifierRange[] { + const esmRanges = parseModuleSpecifiers(code)[0].flatMap((specifier) => + specifier.n === undefined || specifier.d === -2 + ? [] + : [ + { + start: + code[specifier.s] === '"' || code[specifier.s] === "'" + ? specifier.s + 1 + : specifier.s, + end: + code[specifier.e - 1] === '"' || code[specifier.e - 1] === "'" + ? specifier.e - 1 + : specifier.e, + specifier: specifier.n, + requireCondition: false, + }, + ], + ); + return [...esmRanges, ...collectRequireSpecifiers(code)]; +} + +/** Escape one replacement for the quote surrounding its original module specifier. */ +function escapeModuleSpecifier(code: string, range: ModuleSpecifierRange, value: string) { + const escaped = JSON.stringify(value).slice(1, -1); + return code[range.start - 1] === "'" ? escaped.replaceAll("'", "\\'") : escaped; +} + +/** Apply non-overlapping module-specifier replacements without shifting earlier ranges. */ +function replaceModuleSpecifiers( + code: string, + replacements: Array<{ range: ModuleSpecifierRange; value: string }>, +) { + let rewritten = code; + for (const { range, value } of replacements.sort((a, b) => b.range.start - a.range.start)) { + rewritten = + rewritten.slice(0, range.start) + + escapeModuleSpecifier(code, range, value) + + rewritten.slice(range.end); + } + return rewritten; +} + +/** Redirect host-owned imports in transpiled source to the virtual modules. */ +export function rewriteHostSpecifiers(code: string) { + const replacements = collectModuleSpecifiers(code) + .filter(({ specifier }) => specifier in HOST_MODULE_LOADERS) + .map((range) => ({ range, value: `${HOST_MODULE_PREFIX}${range.specifier}` })); + return replaceModuleSpecifiers(code, replacements); +} + +/** Report whether one resolved dependency should pass through Hunk's ESM source hook. */ +function shouldRegisterDependencySource(path: string) { + if (/\.(?:mjs|mts|ts|tsx|jsx)$/i.test(path)) { + return true; + } + if (!/\.js$/i.test(path)) { + return false; + } + + let directory = dirname(path); + while (true) { + const packageJsonPath = join(directory, "package.json"); + if (existsSync(packageJsonPath)) { + try { + return JSON.parse(readFileSync(packageJsonPath, "utf8")).type === "module"; + } catch { + return false; + } + } + + const parent = dirname(directory); + if (parent === directory) { + return false; + } + directory = parent; + } +} + +type RuntimeModuleResolver = (specifier: string, directory: string) => string; + +/** Resolve one extension import in both source and compiled Hunk runtimes. */ +function resolveExtensionDependency( + specifier: string, + importerPath: string, + requireCondition: boolean, + runtimeResolve: RuntimeModuleResolver, +) { + try { + return runtimeResolve(specifier, dirname(importerPath)); + } catch { + try { + const conditions = new Set(["bun", "node", requireCondition ? "require" : "import"]); + const resolved = moduleResolve(specifier, pathToFileURL(importerPath), conditions, false); + return resolved.protocol === "file:" ? fileURLToPath(resolved) : resolved.href; + } catch { + return undefined; + } + } +} + /** - * Match one host-owned specifier in import position in transpiled output. + * Resolve extension-owned imports to filesystem URLs before a compiled Hunk evaluates them. * - * `from "x"` covers static imports and re-exports; `import("x")` and - * `require("x")` cover the dynamic forms. Matching quoted specifiers only in - * these positions keeps a *data* string like `"react"` (say, a language id) - * untouched. + * Bun's source runtime resolves a bare package from the importing extension, but a compiled + * executable resolves only modules embedded at build time. Absolute URLs preserve ordinary + * folder-extension dependency resolution in both modes. Registering each resolved module's + * directory also gives its own imports the same treatment when Bun loads it later. */ -const HOST_SPECIFIER_PATTERN = new RegExp( - `((?:\\bfrom|\\bimport|\\brequire)\\s*\\(?\\s*)(["'])(${Object.keys(HOST_MODULE_LOADERS) - .map((specifier) => specifier.replace(/[/@]/g, "\\$&")) - .join("|")})\\2`, - "g", -); +export function rewriteExtensionDependencySpecifiers( + code: string, + importerPath: string, + runtimeResolve: RuntimeModuleResolver = Bun.resolveSync, +) { + const replacements: Array<{ range: ModuleSpecifierRange; value: string }> = []; -/** Redirect host-owned imports in transpiled source to the virtual modules. */ -export function rewriteHostSpecifiers(code: string) { - return code.replace( - HOST_SPECIFIER_PATTERN, - (_all, lead: string, quote: string, specifier: string) => - `${lead}${quote}${HOST_MODULE_PREFIX}${specifier}${quote}`, - ); + for (const range of collectModuleSpecifiers(code)) { + if (RUNTIME_SPECIFIER_SCHEME.test(range.specifier)) { + continue; + } + + const resolved = resolveExtensionDependency( + range.specifier, + importerPath, + range.requireCondition, + runtimeResolve, + ); + if (!resolved) { + // Preserve Bun's normal diagnostic for an unavailable package or local module. + continue; + } + + if (shouldRegisterDependencySource(resolved)) { + registerSourceRoot(dirname(resolved)); + } + + replacements.push({ + range, + value: + range.requireCondition || RUNTIME_SPECIFIER_SCHEME.test(resolved) + ? resolved + : pathToFileURL(resolved).href, + }); + } + + return replaceModuleSpecifiers(code, replacements); } type TranspilerLoader = "js" | "jsx" | "ts" | "tsx"; @@ -177,7 +387,11 @@ function registerSourceRoot(directory: string) { build.onLoad({ filter }, async (args) => { const source = await Bun.file(args.path).text(); const transpiled = transpilerFor(resolveLoader(args.path)).transformSync(source); - return { contents: rewriteHostSpecifiers(transpiled), loader: "js" }; + const hostRewritten = rewriteHostSpecifiers(transpiled); + return { + contents: rewriteExtensionDependencySpecifiers(hostRewritten, args.path), + loader: "js", + }; }); }, }); From 62b9fbf98b1e0235227cbdcc89b45fd95cf43eca Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sat, 8 Aug 2026 10:02:06 -0400 Subject: [PATCH 5/5] chore(nix): update dependency lockfile --- nix/bun.lock.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nix/bun.lock.nix b/nix/bun.lock.nix index a316fdcef..d3167e317 100644 --- a/nix/bun.lock.nix +++ b/nix/bun.lock.nix @@ -405,6 +405,10 @@ url = "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz"; hash = "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="; }; + "acorn@8.15.0" = fetchurl { + url = "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz"; + hash = "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="; + }; "ansi-escapes@7.3.0" = fetchurl { url = "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz"; hash = "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg=="; @@ -517,6 +521,10 @@ url = "https://registry.npmjs.org/errore/-/errore-0.11.0.tgz"; hash = "sha512-/uJh8o4SYfJAPGSDynpLgKRuRWX5yTSP2BXspHVQu8XmwaX1d6ysxr1cBhjTzC1Um2Xov9BQJ2kigT9lvxHYaA=="; }; + "es-module-lexer@1.7.0" = fetchurl { + url = "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz"; + hash = "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="; + }; "eventemitter3@5.0.4" = fetchurl { url = "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz"; hash = "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw=="; @@ -561,6 +569,10 @@ url = "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz"; hash = "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg=="; }; + "import-meta-resolve@4.2.0" = fetchurl { + url = "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz"; + hash = "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg=="; + }; "is-fullwidth-code-point@5.1.0" = fetchurl { url = "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz"; hash = "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==";