fix(streaming): preserve heading IDs and reference links

packages/comark/src/internal/parse/token-processor.ts

@@ -296,6 +296,8 @@ function processBlockToken(
 ): { node: Node | null; nextIndex: number } {
   const token = tokens[startIndex]
 
+  if (token.type === 'reference') return { node: null, nextIndex: startIndex + 1 }
+
   if (token.type === 'hr') {
     return { node: ['hr', {}] as Node, nextIndex: startIndex + 1 }
   }

packages/comark/src/parse.ts

@@ -124,7 +124,11 @@ export function createMarkdownParser<const TPlugins extends readonly ComarkPlugi
 
       const prevOutput = lastOutput
       const isStartsWithLastInput = markdown.startsWith(lastInput ?? '')
-      if (opts.streaming && prevOutput && isStartsWithLastInput) {
+      // TODO(streaming): `]:` and the heading-tail check force a full reparse so
+      // reference definitions / heading IDs stay correct. Find a way to keep
+      // incremental reuse here (carry env.references + id counters into the tail
+      // parse, or patch completed nodes) without scanning/reparsing the prefix.
+      if (opts.streaming && prevOutput && isStartsWithLastInput && !markdown.includes(']:')) {
         const { remainingMarkdownStartLine, reusedNodes, remainingMarkdown } = extractReusableNodes(
           markdown,
           prevOutput
@@ -133,9 +137,12 @@ export function createMarkdownParser<const TPlugins extends readonly ComarkPlugi
         // If there is no remaining markdown, return the previous output
         if (!remainingMarkdown) return prevOutput
 
-        state.parsedLines = remainingMarkdownStartLine
-        state.markdown = remainingMarkdown
-        state.reusableNodes = reusedNodes
+        // Heading IDs and reference links depend on the full document.
+        if (!/#|(?:^|\n)[^\n]*[=-][ \t]*\r?(?:\n|$)/.test(remainingMarkdown)) {
+          state.parsedLines = remainingMarkdownStartLine
+          state.markdown = remainingMarkdown
+          state.reusableNodes = reusedNodes
+        }
       }
 
       if (typeof autoClose === 'function') {
@@ -191,7 +198,8 @@ export function createMarkdownParser<const TPlugins extends readonly ComarkPlugi
 
       if (opts.streaming) {
         state.tree = {
-          frontmatter: frontmatterText ? frontmatterData : (prevOutput?.frontmatter ?? frontmatterData),
+          frontmatter:
+            frontmatterText || !isStartsWithLastInput ? frontmatterData : (prevOutput?.frontmatter ?? frontmatterData),
           meta: {},
           nodes: [...state.reusableNodes, ...nodes],
         }

packages/comark/test/streaming.test.ts

@@ -3,6 +3,31 @@ import { createMarkdownParser } from 'comark'
 import type { ElementNode } from 'comark'
 
 describe('streaming mode', () => {
+  it.each([false, true])('omits reference definitions with streaming: %s', async (streaming) => {
+    const result = await createMarkdownParser()('# Heading\n\n[ref]: https://example.com\n\n[link][ref]', { streaming })
+    expect(result.nodes).toMatchObject([
+      ['h1', {}, 'Heading'],
+      ['p', {}, ['a', { href: 'https://example.com' }, 'link']],
+    ])
+  })
+
+  it.each([
+    ['duplicate headings', '# Same\n\nIntro\n\n# Same', '\n\nTail'],
+    ['nested headings', '## Parent\n\nIntro\n\n### Child', '\n\nTail'],
+    ['setext headings', 'Same\n====\n\nIntro\n\nSame\n====', '\n\nTail'],
+    ['CRLF setext headings', 'Same\r\n====\r\n\r\nIntro\r\n\r\nSame\r\n====', '\r\n\r\nTail'],
+    ['headings in lists', '# Same\n\nIntro\n\n- # Same', '\n\nTail'],
+    ['existing references', '[ref]: https://example.com\n\n# Heading\n\nIntro\n\n[link][ref]', ' grows'],
+    ['new references', '[link][ref]\n\nIntro\n\nTail', '\n\n[ref]: https://example.com'],
+  ])('matches a full parse after appending to %s', async (_, source, appended) => {
+    const parse = createMarkdownParser()
+    await parse(source, { streaming: true })
+
+    const result = await parse(source + appended, { streaming: true })
+
+    expect(result).toMatchObject(await createMarkdownParser()(source + appended))
+  })
+
   describe('$.line metadata', () => {
     it('preserves position metadata on nodes in streaming mode', async () => {
       const parse = createMarkdownParser()
@@ -190,6 +215,18 @@ describe('streaming mode', () => {
       expect(result1.frontmatter).toEqual({ title: 'Hello' })
       expect(result2.frontmatter).toEqual({ title: 'Hello' })
     })
+
+    it.each([
+      ['without frontmatter', '# Replacement', {}],
+      ['with new frontmatter', '---\ntitle: New\n---\n\n# Replacement', { title: 'New' }],
+    ])('replaces stream frontmatter when the source changes %s', async (_, replacement, frontmatter) => {
+      const parse = createMarkdownParser()
+      await parse('---\ntitle: Old\n---\n\n# Original', { streaming: true })
+
+      const result = await parse(replacement, { streaming: true })
+
+      expect(result.frontmatter).toEqual(frontmatter)
+    })
   })
 
   describe('streaming with MDC components', () => {

test/bundle.test.ts

@@ -67,7 +67,7 @@ describe('package bundle size', { timeout: 60_000 }, () => {
         "@comark/react": "36.8k (74 files)",
         "@comark/svelte": "43.9k (82 files)",
         "@comark/vue": "54.7k (78 files)",
-        "comark": "364k (158 files)",
+        "comark": "365k (158 files)",
       }
     `)
   })