Place every heredoc body and say whether it terminated

README.md

@@ -153,6 +153,11 @@ root: check `errors` on every nested `script` while traversing. A consumer that
 only reads the root `errors` array cannot tell that a substitution body failed
 to parse.
 
+A here-document's `content` is its raw body. `contentPos` and `contentEnd`
+bound that body in the source, and `heredocTerminated` is false when end of
+input ended the body before its delimiter line, which Bash accepts with a
+warning.
+
 ### Print
 
 Basic opinionated printer, does not preserve whitespace or comments (except

src/lexer.ts

@@ -419,11 +419,20 @@ function scanAssignmentPrefix(src: string, start: number, end: number, initialSt
   return state;
 }
 
+interface HereDocTarget {
+  content?: string;
+  heredocQuoted?: boolean;
+  body?: Word;
+  contentPos?: number;
+  contentEnd?: number;
+  heredocTerminated?: boolean;
+}
+
 interface PendingHereDoc {
   delimiter: string;
   strip: boolean;
   quoted: boolean;
-  target?: { content?: string; heredocQuoted?: boolean; body?: Word };
+  target?: HereDocTarget;
 }
 
 const NO_EXPANSIONS: [DeferredCommandExpansion, number][] = [];
@@ -544,6 +553,8 @@ export class Lexer {
   private hasPeek: boolean;
   private pendingHereDocs: PendingHereDoc[] | null;
   private collectedExpansions: [DeferredCommandExpansion, number][] | null;
+  // Whether the last heredoc body scan found its delimiter line, or ran to end of input.
+  private _hereDocTerminated = false;
   _errors: ParseError[] | null = null;
   _buildParts = false;
   // Build processed text while scanning. Off on the normal token path (values
@@ -1166,7 +1177,7 @@ export class Lexer {
     return parts.length > 1 || (parts.length === 1 && parts[0].type !== "Literal") ? parts : null;
   }
 
-  registerHereDocTarget(target: { content?: string; heredocQuoted?: boolean; body?: Word }): void {
+  registerHereDocTarget(target: HereDocTarget): void {
     if (this.pendingHereDocs === null) return;
     for (const hd of this.pendingHereDocs) {
       if (!hd.target) {
@@ -1693,9 +1704,13 @@ export class Lexer {
     if (pending === null || pending.length === 0) return;
     for (const hd of pending) {
       const bodyPos = this.pos;
-      const body = this.readHereDocBody(hd.delimiter, hd.strip);
+      const bodyEnd = this.skipHereDocBody(hd.delimiter, hd.strip);
+      const body = this.src.slice(bodyPos, bodyEnd);
       if (hd.target) {
         hd.target.content = body;
+        hd.target.contentPos = bodyPos;
+        hd.target.contentEnd = bodyEnd;
+        hd.target.heredocTerminated = this._hereDocTerminated;
         if (hd.quoted) {
           hd.target.heredocQuoted = true;
         } else if (body) {
@@ -1707,12 +1722,6 @@ export class Lexer {
     pending.length = 0;
   }
 
-  private readHereDocBody(delimiter: string, strip: boolean): string {
-    const bodyStart = this.pos;
-    const bodyEnd = this.skipHereDocBody(delimiter, strip);
-    return this.src.slice(bodyStart, bodyEnd);
-  }
-
   // Advance past the heredoc body and its delimiter line; return the body end
   // (start of the delimiter line, or srcEnd when delimited by end-of-input).
   // With parenEnds (inside $(...)), a line starting with the delimiter directly
@@ -1762,6 +1771,7 @@ export class Lexer {
       if (lineEnd - lineStart === dLen && src.startsWith(delimiter, lineStart)) {
         const bodyEnd = this.pos;
         this.pos = lineEnd < len ? lineEnd + 1 : lineEnd;
+        this._hereDocTerminated = true;
         return bodyEnd;
       }
 
@@ -1774,13 +1784,15 @@ export class Lexer {
           if (paren !== -1 && paren < this.logicalLineEnd(lineStart, len, !quoted)) {
             const bodyEnd = this.pos;
             this.pos = afterDelim;
+            this._hereDocTerminated = true;
             return bodyEnd;
           }
         }
       }
 
       this.pos = lineEnd < len ? lineEnd + 1 : lineEnd;
     }
+    this._hereDocTerminated = false;
     return this.pos;
   }
 

src/parser.ts

@@ -1437,6 +1437,9 @@ class Parser {
       content: t.content,
       heredocQuoted: undefined,
       body: undefined,
+      contentPos: undefined,
+      contentEnd: undefined,
+      heredocTerminated: undefined,
     };
     if (t.targetEnd > t.targetPos) {
       const heredoc = t.value === "<<" || t.value === "<<-";

src/types.ts

@@ -202,6 +202,11 @@ export interface Redirect {
   content: string | undefined;
   heredocQuoted: boolean | undefined;
   body: Word | undefined;
+  /** The span `content` was sliced from; present whenever `content` is. */
+  contentPos: number | undefined;
+  contentEnd: number | undefined;
+  /** False when end of input ended the body before its delimiter line. */
+  heredocTerminated: boolean | undefined;
 }
 
 export interface Command {

test/heredoc-positions.test.ts

@@ -0,0 +1,70 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { parse } from "../src/parser.ts";
+import type { Command, Redirect } from "../src/types.ts";
+
+const redirects = (src: string): Redirect[] => (parse(src).commands[0].command as Command).redirects;
+const body = (src: string, r: Redirect) => src.slice(r.contentPos!, r.contentEnd!);
+
+test("a heredoc body's span slices its content", () => {
+  for (const src of [
+    "cat <<EOF\nplain\nEOF\n",
+    "cat <<'EOF'\n$quoted\nEOF\n",
+    "cat <<-EOF\n\tindented\n\tEOF\n",
+    "cat <<EOF\nhas $x\nEOF\n",
+    "cat <<EOF\nEOF\n",
+  ]) {
+    const [r] = redirects(src);
+    assert.equal(body(src, r), r.content, src);
+    assert.equal(r.heredocTerminated, true, src);
+  }
+});
+
+test("an expanding body's word and span agree", () => {
+  const [r] = redirects("cat <<EOF\nhas $x\nEOF\n");
+  assert.equal(r.body!.pos, r.contentPos);
+  assert.equal(r.body!.end, r.contentEnd);
+});
+
+test("two heredocs on one line take consecutive spans", () => {
+  const src = "cat <<A <<B\na\nA\nb\nB\n";
+  const [a, b] = redirects(src);
+  assert.equal(body(src, a), "a\n");
+  assert.equal(body(src, b), "b\n");
+  assert.equal(b.contentPos, a.contentEnd! + "A\n".length);
+});
+
+test("a body ended by end of input is unterminated and runs to the end", () => {
+  for (const src of ["cat <<EOF\nnever\n", "cat <<EOF\nnever", "cat <<EOF\n"]) {
+    const [r] = redirects(src);
+    assert.equal(r.heredocTerminated, false, src);
+    assert.equal(r.contentEnd, src.length, src);
+    assert.equal(body(src, r), r.content, src);
+  }
+});
+
+test("an indented, padded, or extended delimiter does not terminate", () => {
+  for (const src of ["cat <<EOF\nx\n  EOF\n", "cat <<EOF\nx\nEOF \n", "cat <<EOF\nx\nEOFX\n"]) {
+    const [r] = redirects(src);
+    assert.equal(r.heredocTerminated, false, src);
+    assert.equal(r.contentEnd, src.length, src);
+  }
+});
+
+test("a heredoc inside a substitution carries its span on the nested script", () => {
+  const src = "echo $(cat <<EOF\ninner\nEOF\n)";
+  const cmd = parse(src).commands[0].command as Command;
+  const part = cmd.suffix[0].parts!.find((p) => p.type === "CommandExpansion")!;
+  assert.equal(part.type, "CommandExpansion");
+  const inner = part.script!.commands[0].command as Command;
+  const [r] = inner.redirects;
+  assert.equal(body(src, r), "inner\n");
+  assert.equal(r.heredocTerminated, true);
+});
+
+test("a redirect that is not a heredoc has no span", () => {
+  const [r] = redirects("cat <file");
+  assert.equal(r.contentPos, undefined);
+  assert.equal(r.contentEnd, undefined);
+  assert.equal(r.heredocTerminated, undefined);
+});