Compile captured closures' own bodies, not just their creation

include/openscad_cpp_evaluator/bytecode.hpp

@@ -276,24 +276,24 @@ struct CompiledChunk {
     // literal's own ClosureSite still needed from beyond `node`'s own scope
     // (i.e. any capture whose resolution target isn't `node` itself). This
     // "bubbling" is what makes a closure nested inside another compiled-
-    // creating closure resolve correctly even though the inner one's body
-    // is never itself compiled (see this struct's own body-compilation
-    // note below): by the time a real, running CompiledChunk executes
-    // Op::MakeClosure for `node`, every one of `captures`'s entries is
-    // guaranteed to resolve against THAT SAME running chunk's own `slots`
-    // array, regardless of how many literal-within-literal levels `node`
-    // was originally nested through in the source.
+    // creating closure resolve correctly: at the OUTERMOST level a given
+    // capture is actually snapshotted (see Op::MakeClosure's own runtime
+    // doc comment, bytecode_vm.cpp), `captures`'s entries resolve against
+    // that running chunk's own `slots` array whenever `targetDecl` matches
+    // its own declaration -- and, when it doesn't (a capture bubbled up
+    // from an even-more-deeply-nested literal), against the enclosing
+    // frame via Evaluator::findUpvalue or, failing that, `ctx.let_`
+    // (rooted at that OUTER closure's own capturedLet whenever it's itself
+    // an escaped invocation) -- regardless of how many literal-within-
+    // literal levels `node` was originally nested through in the source.
     //
-    // `node`'s own BODY is deliberately never itself compiled/registered
-    // once it has any capture need (even an empty one bubbled up from
-    // something it contains) -- it always runs via the ordinary AST
-    // interpreter when invoked (Evaluator::lookupCompiledLiteralChunk finds
-    // no cache entry for it), using Closure::capturedLet exactly like any
-    // other escaping closure the interpreter creates directly. This keeps
-    // the whole feature to "make closure CREATION safe to compile"; a
-    // closure's own body additionally getting compiled too (its captures
-    // would need to read from this snapshot instead of a live-stack walk)
-    // is a natural follow-on, not attempted here.
+    // `node`'s own body IS also registered (Evaluator::
+    // lookupCompiledLiteralChunk finds a real cache entry for it, same as
+    // a zero-capture literal), so a closure invocation can run compiled
+    // too, not just its creation: Op::LoadUpvalue/Op::MakeClosure inside
+    // that body fall back to `ctx.let_` exactly as described above
+    // whenever the live-call-stack walk misses, which is precisely what
+    // happens once this closure has actually escaped its creator.
     struct ClosureSite {
         const oscad::FunctionLiteral* node = nullptr;
         std::vector<UpvalueRef> captures;
@@ -307,6 +307,15 @@ struct CompiledChunk {
         std::vector<std::optional<std::string>> argNames;
     };
 
+    // The FunctionDeclaration/FunctionLiteral this chunk's own body was
+    // compiled from (bytecode_compiler.cpp's compileFunctionLike sets this
+    // to its own `selfDecl` parameter) -- Op::MakeClosure's runtime handler
+    // (bytecode_vm.cpp) compares a ClosureSite capture's own `targetDecl`
+    // against this to tell "genuinely local to the running chunk" (read
+    // `slots` directly) from "bubbled up from a nested literal" (needs the
+    // enclosing-frame/capturedLet fallback) apart.
+    const oscad::ASTNode* selfDecl = nullptr;
+
     std::vector<Param> params;
     std::vector<std::vector<Instruction>> defaultCode;
     std::vector<Instruction> bodyCode;

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "openscad_cpp_evaluator"
-version = "0.7.0"
+version = "0.8.0"
 description = "C++ OpenSCAD evaluator with Python bindings"
 readme = "README.md"
 requires-python = ">=3.12"

src/bytecode_compiler.cpp

@@ -44,11 +44,9 @@ bool isListCompClauseKind(oscad::NodeKind kind) {
 // `owner` itself (targetDecl != owner) -- see ClosureSite's own doc comment
 // (bytecode.hpp) for why this "bubbling" is necessary: a literal nested
 // inside `owner` may need something from further out than `owner`'s own
-// scope, and since that inner literal's body is never itself compiled
-// (only ever invoked via the interpreter once it has any capture need),
-// `owner`'s OWN Op::MakeClosure must snapshot that name too -- otherwise
-// the inner literal's interpreter-resolved capturedLet ancestry, rooted at
-// whatever `owner` itself captured, would simply never contain it.
+// scope, so `owner`'s OWN Op::MakeClosure must snapshot that name too --
+// otherwise the inner literal's own capturedLet, rooted at whatever `owner`
+// itself captured, would simply never contain it.
 // Dedupes by (targetDecl, slot): the same enclosing binding reached through
 // two different nested literals (or the same literal referencing it twice)
 // is still one capture to make.
@@ -67,6 +65,43 @@ void bubbleEscapingCaptures(const CompiledChunk::ClosureSite& site, const oscad:
     }
 }
 
+// True if any instruction in `chunk`'s own body/defaults is Op::LoadFree --
+// i.e. an identifier that resolved to neither a local, a statically-known
+// enclosing upvalue, nor a dyn ($-prefixed) name at compile time (see
+// compileExpr's own Identifier case). A self-referential recursive closure
+// (`let(a = function(x,i) ... a(...) ...)`, the reduce()/accumulate()/
+// while() idiom) hits exactly this: LetOp's own declareLocal for `a` runs
+// only AFTER compiling `a`'s RHS, so `a`'s reference to itself can't
+// resolve as an upvalue at compile time and falls through to LoadFree --
+// Evaluator::evalIdentifier's own ctx.scope->lookupVariable() fallback,
+// which re-evaluates the let-binding's RHS FRESH on every single call,
+// producing a new Closure whose capturedLet is THIS invocation's own
+// ctx.let_ (see expr_eval.cpp's FunctionLiteral case) rather than a stable
+// snapshot. Registering such a chunk for compiled invocation (see the
+// FunctionLiteral case below) would still be functionally correct -- but
+// each recursive call's own capturedLet->openChild() then nests ONE level
+// deeper than the last (confirmed empirically: an O(n) list reduce()
+// degrading to O(n^2) once compiled), since nothing about that repeated
+// fresh-derivation ever re-roots the trail. Excluding a LoadFree-containing
+// chunk from registration leaves it running interpreted exactly as before
+// (this compile-time gap itself isn't fixed here) -- a real, if imperfect,
+// scope for now; closing it -- letting `a` resolve as a genuine upvalue of
+// its own declaring LetOp -- is future work, not bundled into this change.
+bool containsLoadFree(const std::vector<Instruction>& code) {
+    for (const Instruction& ins : code) {
+        if (ins.op == Op::LoadFree) return true;
+    }
+    return false;
+}
+
+bool containsLoadFree(const CompiledChunk& chunk) {
+    if (containsLoadFree(chunk.bodyCode)) return true;
+    for (const auto& defaultCode : chunk.defaultCode) {
+        if (containsLoadFree(defaultCode)) return true;
+    }
+    return false;
+}
+
 // Compile-time name -> slot resolution, mirroring the LetOp/nested-scope
 // shadowing rules callCtx()/letChildCtx() apply at runtime: one frame per
 // LetOp (pushed/popped around its own assignments+body), innermost frame
@@ -306,32 +341,41 @@ class Compiler {
                 for (const auto& nestedSite : literalChunk.closureSites) {
                     bubbleEscapingCaptures(nestedSite, &n, effectiveCaptures);
                 }
-                if (effectiveCaptures.empty()) {
-                    // Closes over nothing at all, even transitively -- the
-                    // existing fast path: a single frozen constant works
-                    // for every invocation, and this literal's own compiled
-                    // chunk (possibly containing its OWN MakeClosure sites
-                    // for anything nested inside IT that closes only over
-                    // ITS OWN scope) is genuinely usable, registered as
-                    // before.
+                // Registered (Evaluator::lookupCompiledLiteralChunk finds
+                // this by node pointer whenever the closure is later
+                // invoked, whether that's a plain PushConst-created value
+                // below or a MakeClosure-snapshotted one) whenever it's
+                // actually safe to run compiled: always for a zero-capture
+                // literal (a single frozen constant, unaffected by anything
+                // below), and for a captures-having one only when its own
+                // body contains no Op::LoadFree -- see containsLoadFree's
+                // own doc comment, above, for why a LoadFree-containing
+                // captures-having closure (the reduce()/accumulate()/
+                // while() self-referential-recursion idiom) must still be
+                // left interpreted: its own capturedLet-chaining, once
+                // registered, degrades an O(n) reduction into O(n^2).
+                // Everything else (Op::LoadUpvalue/Op::MakeClosure inside a
+                // REGISTERED captures-having chunk) is no longer assumed to
+                // resolve only against a still-live creator frame (see
+                // those opcodes' own runtime fallback, bytecode_vm.cpp):
+                // when the live-call-stack walk misses (the creator's frame
+                // is gone -- a genuinely escaped closure), they fall back to
+                // `ctx.let_`, which callCtxFor roots at this exact closure's
+                // own capturedLet snapshot whenever it's invoked -- the same
+                // snapshot `effectiveCaptures` describes below.
+                if (effectiveCaptures.empty() || !containsLoadFree(literalChunk)) {
                     chunk_.nestedLiterals.emplace_back(&n, std::move(literalChunk));
+                }
+                if (effectiveCaptures.empty()) {
+                    // Closes over nothing at all, even transitively -- a
+                    // single frozen constant works for every invocation, no
+                    // per-call snapshot needed.
                     out.push_back(
                         {Op::PushConst, internConst(Value{std::make_shared<const Closure>(Closure{&n, nullptr})}), 0,
                          nullptr});
                     return;
                 }
                 // Escaping-closure support (Op::MakeClosure, bytecode.hpp).
-                // `literalChunk`'s own bytecode is discarded here -- its own
-                // upvalue reads would resolve via the live-call-stack walk
-                // (Op::LoadUpvalue), which is wrong for a closure that
-                // outlives its creator -- so this literal always runs
-                // interpreted when invoked instead (Evaluator::
-                // lookupCompiledLiteralChunk finds no cache entry for it),
-                // using the exact same Closure::capturedLet machinery every
-                // other escaping closure the interpreter creates directly
-                // already relies on. Only `effectiveCaptures` (the names/
-                // slots to snapshot right now) survives from the discarded
-                // compile attempt.
                 int siteIdx = static_cast<int>(chunk_.closureSites.size());
                 chunk_.closureSites.push_back(CompiledChunk::ClosureSite{&n, std::move(effectiveCaptures)});
                 out.push_back({Op::MakeClosure, siteIdx, 0, nullptr});
@@ -896,6 +940,7 @@ bool compileFunctionLike(CompiledChunk& chunk, const oscad::Scope* staticScope,
         if (!p->name->name.empty() && p->name->name[0] == '$') return false;
     }
 
+    chunk.selfDecl = selfDecl;
     Compiler compiler(chunk, staticScope, selfDecl, std::move(enclosing));
     CompileScope bodyScope;
     bodyScope.push();

src/bytecode_vm.cpp

@@ -127,8 +127,22 @@ Value runChunk(Evaluator& ev, const CompiledChunk& chunk, const std::vector<Inst
                 ++pc;
                 break;
             case Op::LoadUpvalue: {
+                // findUpvalue walks the LIVE call stack -- correct as long
+                // as the declaring call is still active, however many
+                // upvalue levels out `uv.targetDecl` sits (an ordinary,
+                // not-yet-escaped nested closure, e.g. reduce()'s own
+                // self-recursive accumulator). If that frame is gone
+                // (this chunk is running as an ESCAPED closure's own body
+                // -- see the FunctionLiteral case's own doc comment,
+                // bytecode_compiler.cpp), fall back to `ctx.let_`: for any
+                // closure invocation with a non-null capturedLet,
+                // callCtxFor roots `ctx.let_` at that exact snapshot, which
+                // is guaranteed (by construction -- effectiveCaptures is a
+                // superset of this chunk's own upvalues) to contain this
+                // name.
                 const CompiledChunk::UpvalueRef& uv = chunk.upvalues[static_cast<size_t>(ins.a)];
                 const Value* v = ev.findUpvalue(uv.targetDecl, uv.slot);
+                if (!v) v = ctx.let_->find(uv.name);
                 stack.push_back(v ? *v : Value{});
                 ++pc;
                 break;
@@ -137,24 +151,36 @@ Value runChunk(Evaluator& ev, const CompiledChunk& chunk, const std::vector<Inst
                 // Builds a FRESH captured environment every time this
                 // instruction actually runs (never a compile-time constant
                 // -- a loop creating one closure per iteration must capture
-                // THAT iteration's own values, not share one snapshot) by
-                // reading each of the site's own captures straight out of
-                // THIS frame's own `slots` -- see ClosureSite's own doc
-                // comment (bytecode.hpp) for why every capture, however
-                // deeply the literal was originally nested, is guaranteed
-                // to resolve against this SAME currently-running chunk.
-                // The resulting Closure::capturedLet is a real, standalone
-                // TrailView (isolated root -- nothing else shares it, and
-                // it owes nothing to `ctx.let_`, which a compiled call
-                // never populates -- see bindCompiledArgs' own doc
-                // comment), so invoking this closure later works through
-                // the EXACT same callCtxFor/capturedLetTrail machinery
-                // every interpreter-created escaping closure already uses,
-                // unchanged.
+                // THAT iteration's own values, not share one snapshot).
+                // Each capture is genuinely local to the CURRENTLY-RUNNING
+                // chunk only when its own `targetDecl` matches this chunk's
+                // `selfDecl` (this literal's direct free-variable
+                // references) -- read straight out of `slots` in that case.
+                // Anything else is a capture bubbled up from a literal
+                // nested even deeper (see ClosureSite's own doc comment,
+                // bytecode.hpp): resolved the same way Op::LoadUpvalue
+                // resolves any upvalue reference, above -- the live call
+                // stack first (still live if THIS closure hasn't itself
+                // escaped yet), `ctx.let_` otherwise (this chunk's own
+                // capturedLet, when it's running as an escaped closure's
+                // body). The resulting Closure::capturedLet is a real,
+                // standalone TrailView (isolated root -- nothing else
+                // shares it, and it owes nothing to `ctx.let_` itself,
+                // which a compiled call never writes to directly -- see
+                // bindCompiledArgs' own doc comment), so invoking this
+                // closure later works through the EXACT same
+                // callCtxFor/capturedLetTrail machinery every interpreter-
+                // created escaping closure already uses, unchanged.
                 const CompiledChunk::ClosureSite& site = chunk.closureSites[static_cast<size_t>(ins.a)];
                 auto capturedTrail = TrailView<Value>::makeRoot();
                 for (const auto& cap : site.captures) {
-                    capturedTrail->set(cap.name, slots[static_cast<size_t>(cap.slot)]);
+                    if (cap.targetDecl == chunk.selfDecl) {
+                        capturedTrail->set(cap.name, slots[static_cast<size_t>(cap.slot)]);
+                        continue;
+                    }
+                    const Value* v = ev.findUpvalue(cap.targetDecl, cap.slot);
+                    if (!v) v = ctx.let_->find(cap.name);
+                    capturedTrail->set(cap.name, v ? *v : Value{});
                 }
                 stack.push_back(Value{std::make_shared<const Closure>(Closure{site.node, std::move(capturedTrail)})});
                 ++pc;

src/user_calls.cpp

@@ -152,8 +152,21 @@ EvalContext Evaluator::callCtxFor(const oscad::ASTNode& decl, EvalContext& ctx,
 
 std::optional<EvalContext> Evaluator::isolatedCallCtxFor(const oscad::ASTNode& declNode, EvalContext& ctx,
                                                           const std::shared_ptr<TrailView<Value>>& capturedLet) {
+    // `declNode.scope()`, not `ctx.scope` -- matches every other call-
+    // resolution function's own `fnScope = decl.scope() ? decl.scope() :
+    // ctx.scope` pattern (evalUserFunction/evalUserFunctionFromBound/
+    // evalFunctionLiteral/evalFunctionLiteralFromBound). Mismatching here
+    // was invisible until a captures-having FunctionLiteral's own body
+    // could ever actually be trampolined into (Op::MakeClosure's own
+    // registration -- bytecode_compiler.cpp's FunctionLiteral case): a
+    // self-referential closure's free-variable resolution can fall through
+    // to evalIdentifier's ctx.scope->lookupVariable() fallback (see its own
+    // doc comment, expr_eval.cpp), which needs the CALLEE's own lexical
+    // scope to walk outward from -- the caller's scope at the call site
+    // has no relation to it at all.
     bool usedChildCtx = false;
-    EvalContext result = callCtxFor(declNode, ctx, ctx.scope, nullptr, nullptr, &usedChildCtx, capturedLet);
+    const oscad::Scope* declScope = declNode.scope() ? declNode.scope() : ctx.scope;
+    EvalContext result = callCtxFor(declNode, ctx, declScope, nullptr, nullptr, &usedChildCtx, capturedLet);
     if (usedChildCtx) return std::nullopt;
     return result;
 }