feat(extensions): support custom syntax grammars

.changeset/extension-syntax-languages.md

@@ -0,0 +1,5 @@
+---
+"hunkdiff": minor
+---
+
+Let extensions register lazy custom syntax grammars for file highlighting.

README.md

@@ -210,8 +210,8 @@ repository's `.hunk/extensions/` (after you explicitly trust that repository),
 and from `--extension <path>` for development. `--no-extensions` turns those off
 for one run; Hunk's own bundled backends (Git, Jujutsu, and Sapling) stay loaded.
 
-A Phase 1 extension can contribute themes and file-extension → language
-mappings, add a VCS backend, rewrite the changeset before review (collapse
+An extension can contribute themes, lazy syntax grammars, and file-extension →
+language mappings, add a VCS backend, rewrite the changeset before review (collapse
 lockfiles, reorder files by review priority), replace the file-navigation
 sidebar with its own React component, react to lifecycle events, and show
 transient messages:

docs/extension-architecture.md

@@ -46,12 +46,16 @@ load issue and costs only that extension. The rules themselves are stated in
 
 ## One registry, one apply path
 
-Registrations (themes, file languages, VCS adapters, changeset transforms,
+Registrations (themes, syntax languages, file languages, VCS adapters, changeset transforms,
 sidebar views, commands, lifecycle/UI events, and inter-extension bus listeners) collect into one
 `ExtensionRegistry` (`src/extensions/types.ts`) and are resolved/applied
 through `src/extensions/apply.ts` on both startup and reload. A factory that
 throws is rolled back to its pre-run registration counts
-(`runExtension.ts`); failures cost a warning, not the session.
+(`runExtension.ts`); failures cost a warning, not the session. Syntax-language
+loaders are registered with Pierre only at this apply boundary, preserving
+factory rollback. Pierre owns the process-wide grammar cache, so an extension
+reload may add a new syntax id but changing or removing an applied grammar needs
+a Hunk restart.
 
 ## Host-served runtime modules
 
@@ -60,10 +64,15 @@ host-served runtime modules (`src/extensions/hostRuntimeModules.ts`): a
 per-extension-directory Bun loader hook transpiles extension source and
 rewrites those specifiers to prefixed virtual modules backed by the host's
 own instances. That identity is what lets `registerSidebarView` components
-render inside the app's React tree with working hooks. The module header
-documents why the obvious alternatives don't work (process-wide specifier
-claims break the host's lazy imports; the loaders resolve lazily so headless
-commands never pay OpenTUI's native-library extraction).
+render inside the app's React tree with working hooks. Other literal imports
+resolve from the extension's own `node_modules` and are rewritten to filesystem
+URLs, including package export/import maps, because Bun's compiled runtime
+cannot resolve packages that were installed after Hunk itself was built. Resolved
+package modules join the same scoped loader path so their dependencies work
+recursively. The module header documents why the obvious alternatives don't
+work (process-wide specifier claims break the host's lazy imports; the loaders
+resolve lazily so headless commands never pay OpenTUI's native-library
+extraction).
 
 ## Sidebar system
 

docs/extensions.md

@@ -188,7 +188,7 @@ cannot mutate the registry mid-session.
 
 ### `hunk.apiVersion`
 
-The API generation this Hunk speaks (currently `2`). Branch on it if you want
+The API generation this Hunk speaks (currently `3`). Branch on it if you want
 one file to support several Hunk versions.
 
 ### `hunk.registerTheme(theme)`
@@ -211,10 +211,55 @@ built-in id. Config-defined themes always win over extension themes for the same
 id; the loser is reported as a startup notice. Extension themes appear in the
 selector after config themes, in load order.
 
+### `hunk.registerSyntaxLanguage(language, loader)`
+
+Register a Shiki-compatible TextMate grammar under a highlighting language id.
+The loader is lazy: Hunk calls it only when Pierre first highlights that
+language. It resolves to an ES-module-shaped object whose `default` export is a
+non-empty grammar array. Every grammar needs at least `name` and `scopeName`;
+normal TextMate fields such as `patterns` and `repository` pass through to
+Shiki.
+
+```ts
+hunk.registerSyntaxLanguage("example-lang", async () => ({
+  default: [
+    {
+      name: "example-lang",
+      scopeName: "source.example-lang",
+      patterns: [{ match: "\\b(example|language)\\b", name: "keyword.control.example-lang" }],
+      repository: {},
+    },
+  ],
+}));
+hunk.registerFileLanguage("example", "example-lang");
+```
+
+A folder extension can keep a generated grammar in a helper module or install a
+language package in its own `node_modules`, then pass its dynamic import directly
+(for example, `() => import("@shikijs/langs/odin")`). Import Pierre through neither
+the extension nor that helper: Hunk forwards the loader to the host's own Pierre
+instance so the grammar reaches the highlighter that renders the review.
+
+Syntax ids are trimmed but remain case-sensitive. `text` and `ansi` are
+reserved. The first extension to register another id wins; later claims are
+skipped with an attributed notice. When registered before Pierre first resolves
+it, a custom id takes precedence over a Pierre-bundled grammar with the same id,
+so choose a new id unless replacing the built-in grammar is deliberate.
+
+Pierre's grammar registry lasts for the Hunk process and cannot safely replace
+or unregister a loader. Reloading the same extension is idempotent, but grammar
+changes, removals, and replacing a grammar already used by this process require
+restarting Hunk. A loader that rejects, takes longer than five seconds, returns
+an invalid module, or supplies a grammar Shiki cannot attach produces one
+attributed warning when first used, and that file falls back to unhighlighted
+text. Fallback results are retried when the file is highlighted again, so a
+transient loader failure does not remain cached for the session.
+
 ### `hunk.registerFileLanguage(extension, language)`
 
 Map a file extension to a syntax-highlighting language. The extension may be
-written with or without a leading dot and is lowercased.
+written with or without a leading dot and is lowercased. Use
+`registerSyntaxLanguage` first when the language is not bundled with Pierre.
 
 ```ts
 hunk.registerFileLanguage(".zig", "zig");

nix/bun.lock.nix

@@ -405,6 +405,10 @@
     url = "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz";
     hash = "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==";
   };
+  "acorn@8.15.0" = fetchurl {
+    url = "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz";
+    hash = "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==";
+  };
   "ansi-escapes@7.3.0" = fetchurl {
     url = "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz";
     hash = "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==";
@@ -517,6 +521,10 @@
     url = "https://registry.npmjs.org/errore/-/errore-0.11.0.tgz";
     hash = "sha512-/uJh8o4SYfJAPGSDynpLgKRuRWX5yTSP2BXspHVQu8XmwaX1d6ysxr1cBhjTzC1Um2Xov9BQJ2kigT9lvxHYaA==";
   };
+  "es-module-lexer@1.7.0" = fetchurl {
+    url = "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz";
+    hash = "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==";
+  };
   "eventemitter3@5.0.4" = fetchurl {
     url = "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz";
     hash = "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==";
@@ -561,6 +569,10 @@
     url = "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz";
     hash = "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==";
   };
+  "import-meta-resolve@4.2.0" = fetchurl {
+    url = "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz";
+    hash = "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==";
+  };
   "is-fullwidth-code-point@5.1.0" = fetchurl {
     url = "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz";
     hash = "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==";

package.json

@@ -109,11 +109,14 @@
   },
   "dependencies": {
     "@pierre/diffs": "1.2.2",
+    "acorn": "8.15.0",
     "bun": "^1.3.14",
     "chokidar": "^4.0.3",
     "commander": "^14.0.3",
     "diff": "^8.0.3",
+    "es-module-lexer": "1.7.0",
     "get-east-asian-width": "^1.5.0",
+    "import-meta-resolve": "4.2.0",
     "shell-quote": "1.9.0",
     "string-width": "^8.2.1",
     "zod": "^4.3.6"

scripts/check-pack.ts

@@ -28,6 +28,7 @@ import type {
   ExtensionFileViewSourceRange,
   ExtensionPaintTheme,
   ExtensionReviewSelection,
+  ExtensionSyntaxLanguageLoader,
   ExtensionVcsAdapter,
   ExtensionVcsDiffInput,
   ExtensionVcsLoadContext,
@@ -49,6 +50,10 @@ export default function (hunk: HunkExtensionAPI) {
     syntaxScopes: { "keyword.operator": "#7fd1ff" },
   };
   hunk.registerTheme(theme);
+  const syntaxLoader: ExtensionSyntaxLanguageLoader = async () => ({
+    default: [{ name: "hunk-pack-fixture", scopeName: "source.hunk-pack-fixture" }],
+  });
+  hunk.registerSyntaxLanguage("hunk-pack-fixture", syntaxLoader);
   hunk.registerFileLanguage(".zig", "zig");
 
   const renderRow = (props: ExtensionFileViewRowComponentProps) => {
@@ -317,7 +322,7 @@ const extensionTypes = readFileSync(
   path.join(repoRoot, "dist", "npm", "extension", "extension-api", "types.d.ts"),
   "utf8",
 );
-if (/^\s*import\b/m.test(extensionTypes)) {
+if (/^\s*import\b/m.test(extensionTypes) || /\bimport\s*\(/m.test(extensionTypes)) {
   throw new Error("The public extension-api/types declaration must remain import-free.");
 }
 for (const removedType of [

src/core/fileLanguage.ts

@@ -1,18 +1,20 @@
 import {
+  getCustomExtensionsVersion,
   getFiletypeFromFileName,
-  setCustomExtension,
+  registerCustomLanguage,
+  replaceCustomExtensions,
+  type LanguageRegistration,
   type SupportedLanguages,
 } from "@pierre/diffs";
+import type { ExtensionSyntaxLanguageLoader } from "../extension-api/types";
 
 // Pierre omits these TypeScript extensions, so register them before lookups or rendering.
 const HUNK_CUSTOM_EXTENSIONS: Record<string, SupportedLanguages> = {
   mts: "typescript",
   cts: "typescript",
 };
 
-for (const [extension, language] of Object.entries(HUNK_CUSTOM_EXTENSIONS)) {
-  setCustomExtension(extension, language);
-}
+replaceCustomExtensions(getCustomExtensionsVersion() + 1, HUNK_CUSTOM_EXTENSIONS);
 
 /**
  * Extensions Hunk itself registers, in Pierre's dotless lowercase form.
@@ -25,15 +27,36 @@ export const BUILT_IN_FILE_LANGUAGE_EXTENSIONS: ReadonlySet<string> = new Set(
   Object.keys(HUNK_CUSTOM_EXTENSIONS),
 );
 
-/**
- * Map one dotless, lowercased file extension to a highlight language.
- *
- * Pierre's language union is closed, but extensions supply plain strings; an
- * unknown language simply fails to match a grammar at render time, which is a
- * better failure than refusing the registration outright.
- */
-export function registerFileLanguage(extension: string, language: string) {
-  setCustomExtension(extension, language as SupportedLanguages);
+const syntaxLanguageFailureReporters = new Map<string, (error: unknown) => void>();
+
+/** Register one lazy extension grammar with Pierre's process-wide highlighter. */
+export function registerSyntaxLanguage(
+  language: string,
+  loader: ExtensionSyntaxLanguageLoader,
+  reportFailure?: (error: unknown) => void,
+) {
+  registerCustomLanguage(language, loader as () => Promise<{ default: LanguageRegistration[] }>);
+  if (reportFailure) {
+    syntaxLanguageFailureReporters.set(language, reportFailure);
+  }
+}
+
+/** Attribute a highlight failure when its language came from an extension. */
+export function reportSyntaxLanguageFailure(language: string | undefined, error: unknown) {
+  if (language) {
+    syntaxLanguageFailureReporters.get(language)?.(error);
+  }
+}
+
+/** Replace extension mappings while preserving Hunk's own protected mappings. */
+export function replaceExtensionFileLanguages(
+  mappings: ReadonlyArray<{ extension: string; language: string }>,
+) {
+  const desired: Record<string, SupportedLanguages> = { ...HUNK_CUSTOM_EXTENSIONS };
+  for (const { extension, language } of mappings) {
+    desired[extension] = language as SupportedLanguages;
+  }
+  replaceCustomExtensions(getCustomExtensionsVersion() + 1, desired);
 }
 
 export { getFiletypeFromFileName };

src/extension-api/index.ts

@@ -85,6 +85,8 @@ export type {
   ExtensionSidebarTheme,
   ExtensionSidebarView,
   ExtensionSidebarViewProps,
+  ExtensionSyntaxGrammar,
+  ExtensionSyntaxLanguageLoader,
   ExtensionThemeConfig,
   ExtensionVcsAdapter,
   ExtensionVcsDetection,

src/extension-api/types.ts

@@ -21,7 +21,7 @@
  * Extensions can branch on `hunk.apiVersion` so a newer Hunk can keep loading
  * older extensions without guessing at their expectations.
  */
-export const HUNK_EXTENSION_API_VERSION = 2;
+export const HUNK_EXTENSION_API_VERSION = 3;
 export type HunkExtensionApiVersion = typeof HUNK_EXTENSION_API_VERSION;
 
 export type ExtensionNotifyType = "info" | "warning" | "error";
@@ -479,6 +479,21 @@ export interface NamedCustomThemeConfig extends CustomThemeConfig {
  */
 export type ExtensionThemeConfig = NamedCustomThemeConfig;
 
+/* -------------------------------------------------------------------------- */
+/* Syntax languages                                                            */
+/* -------------------------------------------------------------------------- */
+
+/** Minimum public shape of one Shiki-compatible TextMate grammar. */
+export interface ExtensionSyntaxGrammar {
+  readonly name: string;
+  readonly scopeName: string;
+}
+
+/** Lazy ES module containing the grammar registrations for one syntax language. */
+export type ExtensionSyntaxLanguageLoader<
+  Grammar extends ExtensionSyntaxGrammar = ExtensionSyntaxGrammar,
+> = () => Promise<{ readonly default: readonly Grammar[] }>;
+
 /* -------------------------------------------------------------------------- */
 /* VCS adapters                                                                */
 /* -------------------------------------------------------------------------- */
@@ -1444,6 +1459,11 @@ export interface HunkExtensionAPI {
   readonly apiVersion: HunkExtensionApiVersion;
   /** Contribute one selectable theme. */
   registerTheme(theme: ExtensionThemeConfig): void;
+  /** Register a lazy Shiki/TextMate grammar under one highlight-language id. */
+  registerSyntaxLanguage<Grammar extends ExtensionSyntaxGrammar>(
+    language: string,
+    loader: ExtensionSyntaxLanguageLoader<Grammar>,
+  ): void;
   /** Map one file extension (with or without a leading dot) to a highlight language. */
   registerFileLanguage(extension: string, language: string): void;
   /** Contribute one additional VCS backend. */