Port Kaleidoscope chapter 9 (debug info) and document chapter 10

LLVMSharp.slnx

@@ -57,6 +57,7 @@
     <Project Path="samples/KaleidoscopeTutorial/Chapter6/Chapter6.csproj" />
     <Project Path="samples/KaleidoscopeTutorial/Chapter7/Chapter7.csproj" />
     <Project Path="samples/KaleidoscopeTutorial/Chapter8/Chapter8.csproj" />
+    <Project Path="samples/KaleidoscopeTutorial/Chapter9/Chapter9.csproj" />
   </Folder>
   <Folder Name="/scripts/">
     <File Path="scripts/build.ps1" />

samples/KaleidoscopeTutorial/Chapter6/Parser.cs

@@ -39,6 +39,8 @@ public class Parser(Lexer lexer, IDictionary<char, int> binaryOpPrecedence) : Ch
     //   ::= 'binary' <op> [precedence] '(' id id ')'
     protected override PrototypeAST? ParsePrototype()
     {
+        int line = CurrentLocation.Line;
+
         string functionName;
         int kind; // 0 = identifier, 1 = unary operator, 2 = binary operator.
         int binaryPrecedence = 30;
@@ -123,7 +125,7 @@ public class Parser(Lexer lexer, IDictionary<char, int> binaryOpPrecedence) : Ch
             return LogErrorProto("Invalid number of operands for operator");
         }
 
-        var prototype = new PrototypeAST(functionName, argNames, IsOperator: kind != 0, binaryPrecedence);
+        var prototype = new PrototypeAST(functionName, argNames, IsOperator: kind != 0, binaryPrecedence) { Line = line };
 
         // Make a user-defined binary operator usable in the input that follows its definition.
         if (prototype.IsBinaryOperator)

samples/KaleidoscopeTutorial/Chapter9/Chapter9.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Chapter7\Chapter7.csproj" />
+    <ProjectReference Include="..\Chapter8\Chapter8.csproj" />
+  </ItemGroup>
+</Project>

samples/KaleidoscopeTutorial/Chapter9/CodeGenVisitor.cs

@@ -0,0 +1,157 @@
+// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. Licensed under the MIT License (MIT). See License.md in the repository root for more information.
+
+using Kaleidoscope.AST;
+using LLVMSharp.Interop;
+
+namespace Kaleidoscope.Chapter9;
+
+/// <summary>
+/// Chapter 9 code generation delta — DWARF debug information. This builds on chapter 7's mutable
+/// variables and adds the metadata a debugger needs: a compile unit and file, a subprogram per
+/// function, a local variable per parameter, and a source location on every expression. The frontend
+/// already stamps each AST node with a <see cref="SourceLocation"/>; here we translate those into
+/// <c>!dbg</c> attachments via the <see cref="LLVMDIBuilderRef"/>.
+/// </summary>
+public sealed class CodeGenVisitor : Chapter7.CodeGenVisitor
+{
+    // DWARF attribute encoding for a 4-byte-aligned IEEE float (DW_ATE_float). Kaleidoscope's only
+    // type is 'double', so a single basic type covers every value, parameter, and return.
+    private const uint DwarfAteFloat = 0x04;
+
+    private LLVMDIBuilderRef _diBuilder;
+    private LLVMMetadataRef _file;
+    private LLVMMetadataRef _doubleType;
+    private LLVMMetadataRef _currentScope;
+
+    /// <summary>
+    /// Supplies the debug-info builder and the file all subprograms are scoped to. The compile unit is
+    /// created and owned by the program; it references this file, so we only need the file here. Must be
+    /// called before the driver generates any code.
+    /// </summary>
+    public void InitializeDebugInfo(LLVMDIBuilderRef diBuilder, LLVMMetadataRef file)
+    {
+        _diBuilder = diBuilder;
+        _file = file;
+    }
+
+    // The debug type for 'double', created lazily so it lands after InitializeDebugInfo.
+    private unsafe LLVMMetadataRef DoubleType
+    {
+        get
+        {
+            if (_doubleType.Handle == IntPtr.Zero)
+            {
+                using var name = new MarshaledString("double");
+                _doubleType = LLVM.DIBuilderCreateBasicType(_diBuilder, name, (UIntPtr)name.Length, 64, DwarfAteFloat, LLVMDIFlags.LLVMDIFlagZero);
+            }
+
+            return _doubleType;
+        }
+    }
+
+    /// <summary>
+    /// Attaches the current expression's source position to the builder so subsequent instructions are
+    /// tagged with a <c>!dbg</c> location. Untracked nodes (or code emitted outside a function) clear
+    /// the location instead, which is what LLVM expects for prologue and synthetic instructions.
+    /// </summary>
+    protected override unsafe void EmitLocation(ExprAST node)
+    {
+        if (_currentScope.Handle == IntPtr.Zero || node.Location.Line == 0)
+        {
+            LLVM.SetCurrentDebugLocation2(Builder, null);
+            return;
+        }
+
+        LLVMMetadataRef location = LLVM.DIBuilderCreateDebugLocation(
+            Module.Context, (uint)node.Location.Line, (uint)node.Location.Column, _currentScope, null);
+        LLVM.SetCurrentDebugLocation2(Builder, location);
+    }
+
+    public override unsafe LLVMValueRef CodegenFunction(FunctionAST node)
+    {
+        // Remember the prototype (so recursive references resolve) then get-or-declare the function.
+        RegisterPrototype(node.Proto);
+        LLVMValueRef function = GetFunction(node.Proto.Name);
+
+        if (function.BasicBlocksCount != 0)
+        {
+            throw new InvalidOperationException($"Function '{node.Proto.Name}' cannot be redefined");
+        }
+
+        LLVMBasicBlockRef entry = function.AppendBasicBlock("entry");
+        Builder.PositionAtEnd(entry);
+
+        // Describe the function to the debugger and make its subprogram the active scope.
+        uint line = (uint)(node.Proto.Line == 0 ? 1 : node.Proto.Line);
+        LLVMMetadataRef subprogram = CreateFunctionScope(node.Proto, line);
+        LLVM.SetSubprogram(function, subprogram);
+        _currentScope = subprogram;
+
+        // The prologue (parameter spills) should not have a source location.
+        LLVM.SetCurrentDebugLocation2(Builder, null);
+        CreateDebugParameterBindings(function, node.Proto, entry, subprogram, line);
+
+        try
+        {
+            LLVMValueRef body = Codegen(node.Body);
+            Builder.BuildRet(body);
+            function.VerifyFunction(LLVMVerifierFailureAction.LLVMPrintMessageAction);
+            return function;
+        }
+        catch
+        {
+            // Remove the half-built function so the driver can keep going after an error.
+            function.DeleteFunction();
+            throw;
+        }
+        finally
+        {
+            _currentScope = default;
+        }
+    }
+
+    // Creates the DISubprogram (and its subroutine type) describing one function definition.
+    private LLVMMetadataRef CreateFunctionScope(PrototypeAST proto, uint line)
+    {
+        LLVMMetadataRef subroutineType = CreateSubroutineType(proto.Arguments.Count);
+
+        return _diBuilder.CreateFunction(
+            _file, proto.Name, LinkageName: "", _file, line, subroutineType,
+            IsLocalToUnit: 1, IsDefinition: 1, ScopeLine: line, LLVMDIFlags.LLVMDIFlagPrototyped, IsOptimized: 0);
+    }
+
+    // Element 0 is the return type; the rest are the parameters. Every value is a 'double'.
+    private LLVMMetadataRef CreateSubroutineType(int argumentCount)
+    {
+        var parameterTypes = new LLVMMetadataRef[argumentCount + 1];
+        Array.Fill(parameterTypes, DoubleType);
+        return _diBuilder.CreateSubroutineType(_file, parameterTypes, LLVMDIFlags.LLVMDIFlagZero);
+    }
+
+    // Chapter 7 already spills every parameter into an alloca; here we additionally describe each one
+    // to the debugger with a DILocalVariable and a dbg.declare record pointing at its slot.
+    private unsafe void CreateDebugParameterBindings(LLVMValueRef function, PrototypeAST proto, LLVMBasicBlockRef entry, LLVMMetadataRef scope, uint line)
+    {
+        NamedValues.Clear();
+
+        LLVMMetadataRef emptyExpression = LLVM.DIBuilderCreateExpression(_diBuilder, null, (UIntPtr)0);
+        LLVMMetadataRef declareLocation = LLVM.DIBuilderCreateDebugLocation(Module.Context, line, 0, scope, null);
+
+        for (int i = 0; i < proto.Arguments.Count; i++)
+        {
+            string name = proto.Arguments[i];
+
+            LLVMValueRef alloca = CreateEntryBlockAlloca(function, name);
+            Builder.BuildStore(function.GetParam((uint)i), alloca);
+
+            using var marshaledName = new MarshaledString(name);
+            LLVMMetadataRef variable = LLVM.DIBuilderCreateParameterVariable(
+                _diBuilder, scope, marshaledName, (UIntPtr)marshaledName.Length, (uint)(i + 1), _file, line,
+                DoubleType, AlwaysPreserve: 1, LLVMDIFlags.LLVMDIFlagZero);
+
+            _ = LLVM.DIBuilderInsertDeclareRecordAtEnd(_diBuilder, alloca, variable, emptyExpression, declareLocation, entry);
+
+            NamedValues[name] = alloca;
+        }
+    }
+}

samples/KaleidoscopeTutorial/Chapter9/Program.cs

@@ -0,0 +1,73 @@
+// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. Licensed under the MIT License (MIT). See License.md in the repository root for more information.
+
+using Kaleidoscope;
+using Kaleidoscope.Chapter8;
+using Kaleidoscope.Chapter9;
+using LLVMSharp.Interop;
+
+// Chapter 9 is chapter 8's batch compiler with DWARF debug information layered on top. It reads a whole
+// Kaleidoscope program, generates every definition into one module — now carrying a compile unit, a
+// subprogram per function, local variables for parameters, and a source location on every expression —
+// and emits a native object file that a debugger can step through.
+var binaryOpPrecedence = new Dictionary<char, int>
+{
+    ['='] = 2,
+    ['<'] = 10,
+    ['+'] = 20,
+    ['-'] = 20,
+    ['*'] = 40,
+};
+
+string sourceName = args.Length > 0 ? Path.GetFileName(args[0]) : "stdin.ks";
+string sourceDirectory = args.Length > 0 ? (Path.GetDirectoryName(Path.GetFullPath(args[0])) ?? ".") : ".";
+string outputPath = args.Length > 1 ? args[1] : "output.o";
+
+using TextReader reader = args.Length > 0 ? new StreamReader(args[0]) : Console.In;
+
+var lexer = new Lexer(reader, binaryOpPrecedence);
+var parser = new Kaleidoscope.Chapter7.Parser(lexer, binaryOpPrecedence);
+var visitor = new CodeGenVisitor();
+
+LlvmSupport.EnsureTargetsInitialized();
+LLVMTargetMachineRef targetMachine = LlvmSupport.CreateHostTargetMachine();
+
+LLVMContextRef context = LLVMContextRef.Create();
+LLVMModuleRef module = context.CreateModuleWithName("KaleidoscopeModule");
+LlvmSupport.PrepareModuleForEmit(module, targetMachine);
+
+// Tell the backend a module-level debug-info version is present. DWARF is the default on the tutorial's
+// targets; on Windows/CodeView these flags are still valid and harmless.
+module.AddModuleFlag("Debug Info Version", LLVMModuleFlagBehavior.LLVMModuleFlagBehaviorWarning, LLVM.DebugMetadataVersion());
+module.AddModuleFlag("Dwarf Version", LLVMModuleFlagBehavior.LLVMModuleFlagBehaviorWarning, 4u);
+
+LLVMDIBuilderRef diBuilder = module.CreateDIBuilder();
+LLVMMetadataRef file = diBuilder.CreateFile(sourceName, sourceDirectory);
+
+// Kaleidoscope has no notion of C, but claiming C lets debuggers apply sensible defaults, matching the
+// upstream tutorial.
+LLVMMetadataRef compileUnit = diBuilder.CreateCompileUnit(
+    LLVMDWARFSourceLanguage.LLVMDWARFSourceLanguageC, file, "Kaleidoscope Compiler",
+    IsOptimized: 0, Flags: "", RuntimeVersion: 0, SplitName: "", LLVMDWARFEmissionKind.LLVMDWARFEmissionFull,
+    DWOld: 0, SplitDebugInlining: 0, DebugInfoForProfiling: 0, SysRoot: "", SDK: "");
+
+visitor.InitializeDebugInfo(diBuilder, file);
+
+var driver = new ObjectFileDriver(lexer, parser, visitor, module);
+driver.Run();
+
+// Resolve all temporary debug metadata now that every function has been generated.
+diBuilder.DIBuilderFinalize();
+
+Console.WriteLine("=== Module IR ===");
+Console.Write(module.PrintToString());
+Console.WriteLine();
+
+if (targetMachine.TryEmitToFile(module, outputPath, LLVMCodeGenFileType.LLVMObjectFile, out string message))
+{
+    Console.WriteLine($"Wrote object file to '{outputPath}' (target {module.Target}).");
+}
+else
+{
+    Console.Error.WriteLine($"Failed to emit object file: {message}");
+    Environment.Exit(1);
+}

samples/KaleidoscopeTutorial/Kaleidoscope.Common/Ast/ExprAST.cs

@@ -3,7 +3,14 @@
 namespace Kaleidoscope.AST;
 
 /// <summary>Base class for all expression nodes.</summary>
-public abstract record ExprAST;
+public abstract record ExprAST
+{
+    /// <summary>
+    /// Where this node came from in the source. The parser stamps it (chapter 9); a
+    /// <see cref="SourceLocation.Line"/> of 0 means untracked, and earlier chapters simply ignore it.
+    /// </summary>
+    public SourceLocation Location { get; init; }
+}
 
 /// <summary>A numeric literal, e.g. <c>1.0</c> (chapter 3).</summary>
 public sealed record NumberExprAST(double Value) : ExprAST;

samples/KaleidoscopeTutorial/Kaleidoscope.Common/Ast/PrototypeAST.cs

@@ -16,6 +16,9 @@ public sealed record PrototypeAST(
 
     public bool IsBinaryOperator => IsOperator && Arguments.Count == 2;
 
+    /// <summary>Source line the definition starts on, stamped by the parser (chapter 9); 0 if untracked.</summary>
+    public int Line { get; init; }
+
     /// <summary>The operator character for an operator prototype (the last char of the name).</summary>
     public char OperatorName => Name[^1];
 }

samples/KaleidoscopeTutorial/Kaleidoscope.Common/Ast/SourceLocation.cs

@@ -0,0 +1,9 @@
+// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. Licensed under the MIT License (MIT). See License.md in the repository root for more information.
+
+namespace Kaleidoscope.AST;
+
+/// <summary>
+/// A 1-based source position (chapter 9). A <see cref="Line"/> of 0 means "no location was recorded",
+/// which is how the code generator tells tracked nodes apart from untracked ones.
+/// </summary>
+public readonly record struct SourceLocation(int Line, int Column);

samples/KaleidoscopeTutorial/Kaleidoscope.Common/CodeGenVisitorBase.cs

@@ -34,18 +34,32 @@ public void SetModule(LLVMModuleRef module)
     }
 
     /// <summary>Generates code for an expression, dispatching to the chapter-specific override.</summary>
-    public LLVMValueRef Codegen(ExprAST node) => node switch
+    public LLVMValueRef Codegen(ExprAST node)
     {
-        NumberExprAST n => CodegenNumber(n),
-        VariableExprAST n => CodegenVariable(n),
-        BinaryExprAST n => CodegenBinary(n),
-        UnaryExprAST n => CodegenUnary(n),
-        CallExprAST n => CodegenCall(n),
-        IfExprAST n => CodegenIf(n),
-        ForExprAST n => CodegenFor(n),
-        VarExprAST n => CodegenVar(n),
-        _ => throw Unsupported(node),
-    };
+        // Chapter 9 overrides this to attach a debug location before the node's code is emitted.
+        EmitLocation(node);
+
+        return node switch
+        {
+            NumberExprAST n => CodegenNumber(n),
+            VariableExprAST n => CodegenVariable(n),
+            BinaryExprAST n => CodegenBinary(n),
+            UnaryExprAST n => CodegenUnary(n),
+            CallExprAST n => CodegenCall(n),
+            IfExprAST n => CodegenIf(n),
+            ForExprAST n => CodegenFor(n),
+            VarExprAST n => CodegenVar(n),
+            _ => throw Unsupported(node),
+        };
+    }
+
+    /// <summary>
+    /// Hook called for every expression just before its code is generated. The debug-info chapter
+    /// overrides it to set the builder's current debug location; other chapters leave it a no-op.
+    /// </summary>
+    protected virtual void EmitLocation(ExprAST node)
+    {
+    }
 
     /// <summary>Emits a declaration for an <c>extern</c> and remembers it for later modules.</summary>
     public LLVMValueRef CodegenExtern(PrototypeAST node)