diff --git a/src/engine/BinParser.v3 b/src/engine/BinParser.v3 index 515e55d03..e62c61622 100644 --- a/src/engine/BinParser.v3 +++ b/src/engine/BinParser.v3 @@ -1,6 +1,9 @@ // Copyright 2019 Ben L. Titzer. All rights reserved. // See LICENSE for details of Apache 2.0 license. +// Fast-call opcodes index one dispatch table shared by every module in the process. +var fast_call_slots_used = 0; + // Parses the binary format of WebAssembly and produces an in-memory module // representation. The parser uses a "push" model which allows the creator to // input segments of data into the parser incrementally, e.g. if streaming it @@ -546,6 +549,22 @@ class BinParser(extensions: Extension.set, limits: Limits, err: ErrorGen, filena } var decl = readExportWithoutName(); module.exports.put(name, decl); + // assign fast functions + if (FastIntTuning.useFastFunctions && Strings.startsWith(name, "fast:") && FuncDecl.?(decl)) { + def fdecl = FuncDecl.!(decl); + def fast_funcs = module.fast_funcs; + if (fdecl.fast_call_idx >= 0) { + err.rel(decoder, pt).DuplicateFastExport(fdecl.func_index); + } else if (fdecl.imported()) { + err.rel(decoder, pt).ImportedFastExport(fdecl.func_index); + } else if (fast_call_slots_used >= Opcodes.FAST_CALL_OPCODES) { + // No fast-call slot available; fall back to a regular call for this function. + if (Trace.validation) Trace.OUT.put1(" dropping fast: export, no FAST_CALL slots remain (max %d)\n", Opcodes.FAST_CALL_OPCODES); + } else { + fdecl.fast_call_idx = fast_call_slots_used++; + fast_funcs.put(fdecl); + } + } } def readExportWithoutName() -> Decl { var pt = decoder.pos; diff --git a/src/engine/BytecodeIterator.v3 b/src/engine/BytecodeIterator.v3 index 379e2d3de..c655c5a06 100644 --- a/src/engine/BytecodeIterator.v3 +++ b/src/engine/BytecodeIterator.v3 @@ -785,6 +785,48 @@ class BytecodeIterator { RESUME_THROW => v.visit_RESUME_THROW(read_CONT(), read_TAG(), read_HANDLERS()); RESUME_THROW_REF => v.visit_RESUME_THROW_REF(read_CONT(), read_HANDLERS()); SWITCH => v.visit_SWITCH(read_CONT(), read_TAG()); + + // replacing CALL with FAST_CALL does not touch the operand so that the original function can still be recovered + FAST_CALL0 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL1 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL2 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL3 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL4 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL5 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL6 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL7 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL8 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL9 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL10 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL11 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL12 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL13 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL14 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL15 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL16 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL17 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL18 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL19 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL20 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL21 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL22 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL23 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL24 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL25 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL26 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL27 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL28 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL29 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL30 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL31 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL32 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL33 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL34 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL35 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL36 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL37 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL38 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); + FAST_CALL39 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC()); } } def trace(out: StringBuilder, module: Module, tracer: InstrTracer) { diff --git a/src/engine/CodeValidator.v3 b/src/engine/CodeValidator.v3 index cd7fb116a..bfef64fe7 100644 --- a/src/engine/CodeValidator.v3 +++ b/src/engine/CodeValidator.v3 @@ -49,6 +49,23 @@ class CodeValidator(extensions: Extension.set, limits: Limits, module: Module, e codeptr.startAddr = d.startAddr; return Metrics.validate_time_us.run(validateExpr0, (expected, d)); } + private def checkFastFunctionOpcodes(func: FuncDecl) { + var bi = BytecodeIterator.new().reset(func); + while (bi.more()) { + var reason: string; + match (bi.current()) { + RETURN_CALL, RETURN_CALL_INDIRECT, RETURN_CALL_REF => + reason = "a tail call instruction"; + TRY, CATCH, THROW, RETHROW, THROW_REF, DELEGATE, CATCH_ALL, TRY_TABLE => + reason = "an exception handling instruction"; + CONT_NEW, CONT_BIND, SUSPEND, RESUME, RESUME_THROW, RESUME_THROW_REF, SWITCH => + reason = "a stack switching instruction"; + _ => ; + } + if (reason != null) return err.rel(codeptr, bi.pc).IllegalFastFunctionOpcode(func.func_index, reason); + bi.next(); + } + } private def validateFunc0(func: FuncDecl) -> CodeValidatorResult { // Reset internal state. if (Trace.validation) OUT.put2("==== begin validate: func %q: %q ========================", func.render(module.names, _), func.sig.render).ln(); @@ -87,6 +104,10 @@ class CodeValidator(extensions: Extension.set, limits: Limits, module: Module, e } else OUT.puts(" No reachable handlers").ln(); } + // Fast functions are compiled into dispatch-table handlers that own no frame, + // so they cannot use these instruction groups. + if (err.ok() && FastIntTuning.useFastFunctions && func.fast_call_idx >= 0) checkFastFunctionOpcodes(func); + // Extract sidetable. if (!revalidating) func.sidetable = ctlxfer.extract(); Debug.keepAlive(func.sidetable.entries); @@ -420,6 +441,14 @@ class CodeValidator(extensions: Extension.set, limits: Limits, module: Module, e var func = parser.readFuncRef(); if (func == null) return; checkSignature(func.sig); + + // fast call: if function is exported with fast name, replace the bytecode with FAST_CALL + // only replace if the LEB encoding is minimal + if (FastIntTuning.useFastFunctions && func.fast_call_idx >= 0 + && (codeptr.pos - opcode_pos - 1) == Opcodes.minimalUleb32Length(func.func_index)) { + if (Trace.validation) Trace.OUT.put1(" replacing with FAST_CALL%d\n", func.fast_call_idx); + this.func.replaceCall(opcode_pos, func.fast_call_idx); + } } CALL_INDIRECT => { var sig = parser.readSigRef(); diff --git a/src/engine/Module.v3 b/src/engine/Module.v3 index 84acdb813..6b21cecaa 100644 --- a/src/engine/Module.v3 +++ b/src/engine/Module.v3 @@ -17,6 +17,7 @@ class Module(filename: string) { def exports = Vector<(string, Decl)>.new(); def elems = Vector.new(); def data = Vector.new(); + def fast_funcs = Vector.new(); def custom_sections = Vector.new(); def environments = Vector.new(); var probes: Array>; @@ -149,6 +150,8 @@ class FuncDecl(sig_index: int) extends Decl { var target_code: TargetCode; var tierup_trigger: int = int.max; var handlers = FuncHandlerInfo.new(); + var fast_target_code: TargetCode; + var fast_call_idx: int = -1; def render(names: NameSection, buf: StringBuilder) -> StringBuilder { var name = if (names != null, names.getFuncName(func_index)); @@ -160,6 +163,7 @@ class FuncDecl(sig_index: int) extends Decl { var tc: TargetCode; var tr: TargetCode; target_code = tc; // reset target code as well + fast_target_code = tc; sidetable = Sidetables.NO_SIDETABLE; cbd_sidetable = null; } @@ -174,6 +178,11 @@ class FuncDecl(sig_index: int) extends Decl { if (cur_bytecode == orig_bytecode) return; cur_bytecode[pc] = orig_bytecode[pc]; } + def replaceCall(pc: int, idx: int) { + // copy bytecode for modification + if (cur_bytecode == orig_bytecode) orig_bytecode = Arrays.dup(orig_bytecode); + cur_bytecode[pc] = byte.!(Opcodes.indexToFastCall(idx).code); + } def reset() -> this { if (cur_bytecode == orig_bytecode) return; ArrayUtil.copyInto(cur_bytecode, 0, orig_bytecode, 0, orig_bytecode.length); @@ -189,6 +198,7 @@ class FuncDecl(sig_index: int) extends Decl { n.sidetable = this.sidetable; n.num_locals = this.num_locals; n.target_code = this.target_code; + n.fast_target_code = this.fast_target_code; return n; } def findExHandler(instance: Instance, tag: Tag, throw_pc: int) -> ExHandler { diff --git a/src/engine/Opcodes.v3 b/src/engine/Opcodes.v3 index f5dc7135e..149c3fdd1 100644 --- a/src/engine/Opcodes.v3 +++ b/src/engine/Opcodes.v3 @@ -615,8 +615,51 @@ enum Opcode(prefix: u8, code: u16, mnemonic: string, imms: Array, sig: RESUME_THROW (0x00, 0xE4, "resume_throw", imm.CONT_TAG_HANDLE, null), RESUME_THROW_REF (0x00, 0xE5, "resume_throw_ref", imm.CONT_HANDLE, null), SWITCH (0x00, 0xE6, "switch", imm.CONT_TAG, null) + + // fast call instructions + FAST_CALL0 (0x00, 0xC5, "fast_call0", imm.FUNC, null), + FAST_CALL1 (0x00, 0xC6, "fast_call1", imm.FUNC, null), + FAST_CALL2 (0x00, 0xC7, "fast_call2", imm.FUNC, null), + FAST_CALL3 (0x00, 0xC8, "fast_call3", imm.FUNC, null), + FAST_CALL4 (0x00, 0xC9, "fast_call4", imm.FUNC, null), + FAST_CALL5 (0x00, 0xCA, "fast_call5", imm.FUNC, null), + FAST_CALL6 (0x00, 0xCB, "fast_call6", imm.FUNC, null), + FAST_CALL7 (0x00, 0xCC, "fast_call7", imm.FUNC, null), + FAST_CALL8 (0x00, 0xCD, "fast_call8", imm.FUNC, null), + FAST_CALL9 (0x00, 0xCE, "fast_call9", imm.FUNC, null), + FAST_CALL10 (0x00, 0xCF, "fast_call10", imm.FUNC, null), + FAST_CALL11 (0x00, 0xD7, "fast_call11", imm.FUNC, null), + FAST_CALL12 (0x00, 0xD8, "fast_call12", imm.FUNC, null), + FAST_CALL13 (0x00, 0xD9, "fast_call13", imm.FUNC, null), + FAST_CALL14 (0x00, 0xDA, "fast_call14", imm.FUNC, null), + FAST_CALL15 (0x00, 0xDB, "fast_call15", imm.FUNC, null), + FAST_CALL16 (0x00, 0xDC, "fast_call16", imm.FUNC, null), + FAST_CALL17 (0x00, 0xDD, "fast_call17", imm.FUNC, null), + FAST_CALL18 (0x00, 0xDE, "fast_call18", imm.FUNC, null), + FAST_CALL19 (0x00, 0xDF, "fast_call19", imm.FUNC, null), + FAST_CALL20 (0x00, 0xE7, "fast_call20", imm.FUNC, null), + FAST_CALL21 (0x00, 0xE8, "fast_call21", imm.FUNC, null), + FAST_CALL22 (0x00, 0xE9, "fast_call22", imm.FUNC, null), + FAST_CALL23 (0x00, 0xEA, "fast_call23", imm.FUNC, null), + FAST_CALL24 (0x00, 0xEB, "fast_call24", imm.FUNC, null), + FAST_CALL25 (0x00, 0xEC, "fast_call25", imm.FUNC, null), + FAST_CALL26 (0x00, 0xED, "fast_call26", imm.FUNC, null), + FAST_CALL27 (0x00, 0xEE, "fast_call27", imm.FUNC, null), + FAST_CALL28 (0x00, 0xEF, "fast_call28", imm.FUNC, null), + FAST_CALL29 (0x00, 0xF2, "fast_call29", imm.FUNC, null), + FAST_CALL30 (0x00, 0xF3, "fast_call30", imm.FUNC, null), + FAST_CALL31 (0x00, 0xF4, "fast_call31", imm.FUNC, null), + FAST_CALL32 (0x00, 0xF5, "fast_call32", imm.FUNC, null), + FAST_CALL33 (0x00, 0xF6, "fast_call33", imm.FUNC, null), + FAST_CALL34 (0x00, 0xF7, "fast_call34", imm.FUNC, null), + FAST_CALL35 (0x00, 0xF8, "fast_call35", imm.FUNC, null), + FAST_CALL36 (0x00, 0xF9, "fast_call36", imm.FUNC, null), + FAST_CALL37 (0x00, 0xFA, "fast_call37", imm.FUNC, null), + FAST_CALL38 (0x00, 0x17, "fast_call38", imm.FUNC, null), + FAST_CALL39 (0x00, 0x27, "fast_call39", imm.FUNC, null), } + // Enumeration of the different kinds of immediates to opcodes. enum ImmKind { ARRAY_TYPE_INDEX, // ARRAYT @@ -806,9 +849,16 @@ component Opcodes { def code_pages = [page_FB, page_FC, page_FD, page_FE]; def var longestName: int; def var num_subpages: int; + def FAST_CALL_OPCODES = 40; + def var fast_calls: Array; private var nameMap: HashMap; new() { + + fast_calls = Array.new(FAST_CALL_OPCODES); + for (i < FAST_CALL_OPCODES) { + fast_calls[i] = indexToFastCall(i); + } for (op in Opcode) { if (op == Opcode.INVALID) continue; init(op); @@ -817,6 +867,7 @@ component Opcodes { attributes[InternalOpcode.PROBE.code] = OpcodeAttribute.INTERNAL | OpcodeAttribute.PROBE; attributes[InternalOpcode.WHAMM_PROBE.code] = OpcodeAttribute.INTERNAL | OpcodeAttribute.PROBE; attributes[InternalOpcode.BREAK_PROBE.code] = OpcodeAttribute.INTERNAL | OpcodeAttribute.PROBE; + for (op in fast_calls) attributes[op.tag] = OpcodeAttribute.INTERNAL; for (op in [Opcode.END, Opcode.I32_CONST, Opcode.I64_CONST, Opcode.F32_CONST, Opcode.F64_CONST, Opcode.GLOBAL_GET, Opcode.REF_NULL, Opcode.REF_FUNC, Opcode.STRUCT_NEW, Opcode.STRUCT_NEW_DEFAULT, @@ -1129,6 +1180,108 @@ component Opcodes { } } } + def minimalUleb32Length(val: int) -> int { + var len = 1, n = val; + while (n >= 0x80) { + len++; + n >>= 7; + } + return len; + } + def indexToFastCall(index: int) -> Opcode { + var op: Opcode; + match (index) { + 0 => op = Opcode.FAST_CALL0; + 1 => op = Opcode.FAST_CALL1; + 2 => op = Opcode.FAST_CALL2; + 3 => op = Opcode.FAST_CALL3; + 4 => op = Opcode.FAST_CALL4; + 5 => op = Opcode.FAST_CALL5; + 6 => op = Opcode.FAST_CALL6; + 7 => op = Opcode.FAST_CALL7; + 8 => op = Opcode.FAST_CALL8; + 9 => op = Opcode.FAST_CALL9; + 10 => op = Opcode.FAST_CALL10; + 11 => op = Opcode.FAST_CALL11; + 12 => op = Opcode.FAST_CALL12; + 13 => op = Opcode.FAST_CALL13; + 14 => op = Opcode.FAST_CALL14; + 15 => op = Opcode.FAST_CALL15; + 16 => op = Opcode.FAST_CALL16; + 17 => op = Opcode.FAST_CALL17; + 18 => op = Opcode.FAST_CALL18; + 19 => op = Opcode.FAST_CALL19; + 20 => op = Opcode.FAST_CALL20; + 21 => op = Opcode.FAST_CALL21; + 22 => op = Opcode.FAST_CALL22; + 23 => op = Opcode.FAST_CALL23; + 24 => op = Opcode.FAST_CALL24; + 25 => op = Opcode.FAST_CALL25; + 26 => op = Opcode.FAST_CALL26; + 27 => op = Opcode.FAST_CALL27; + 28 => op = Opcode.FAST_CALL28; + 29 => op = Opcode.FAST_CALL29; + 30 => op = Opcode.FAST_CALL30; + 31 => op = Opcode.FAST_CALL31; + 32 => op = Opcode.FAST_CALL32; + 33 => op = Opcode.FAST_CALL33; + 34 => op = Opcode.FAST_CALL34; + 35 => op = Opcode.FAST_CALL35; + 36 => op = Opcode.FAST_CALL36; + 37 => op = Opcode.FAST_CALL37; + 38 => op = Opcode.FAST_CALL38; + 39 => op = Opcode.FAST_CALL39; + _ => System.error("indexToFastCall", "out of range"); + } + return op; + } + def fastCallToIndex(op: Opcode) -> int { + var idx: int; + match (op) { + FAST_CALL0 => idx = 0; + FAST_CALL1 => idx = 1; + FAST_CALL2 => idx = 2; + FAST_CALL3 => idx = 3; + FAST_CALL4 => idx = 4; + FAST_CALL5 => idx = 5; + FAST_CALL6 => idx = 6; + FAST_CALL7 => idx = 7; + FAST_CALL8 => idx = 8; + FAST_CALL9 => idx = 9; + FAST_CALL10 => idx = 10; + FAST_CALL11 => idx = 11; + FAST_CALL12 => idx = 12; + FAST_CALL13 => idx = 13; + FAST_CALL14 => idx = 14; + FAST_CALL15 => idx = 15; + FAST_CALL16 => idx = 16; + FAST_CALL17 => idx = 17; + FAST_CALL18 => idx = 18; + FAST_CALL19 => idx = 19; + FAST_CALL20 => idx = 20; + FAST_CALL21 => idx = 21; + FAST_CALL22 => idx = 22; + FAST_CALL23 => idx = 23; + FAST_CALL24 => idx = 24; + FAST_CALL25 => idx = 25; + FAST_CALL26 => idx = 26; + FAST_CALL27 => idx = 27; + FAST_CALL28 => idx = 28; + FAST_CALL29 => idx = 29; + FAST_CALL30 => idx = 30; + FAST_CALL31 => idx = 31; + FAST_CALL32 => idx = 32; + FAST_CALL33 => idx = 33; + FAST_CALL34 => idx = 34; + FAST_CALL35 => idx = 35; + FAST_CALL36 => idx = 36; + FAST_CALL37 => idx = 37; + FAST_CALL38 => idx = 38; + FAST_CALL39 => idx = 39; + _ => System.error("fastCallToIndex", "not a FAST_CALL instruction"); + } + return idx; + } } // Renders instructions as text. diff --git a/src/engine/compiler/CompilerOptions.v3 b/src/engine/compiler/CompilerOptions.v3 index 7b9ae7329..e645f3e3e 100644 --- a/src/engine/compiler/CompilerOptions.v3 +++ b/src/engine/compiler/CompilerOptions.v3 @@ -33,7 +33,7 @@ component CompilerOptions { .onSet(fun v => void(SpcTuning.maxInlineBytecodeSize = v)); group.newIntOption("inline-max-params", SpcTuning.maxInlineParams, "Maximum number of parameters of a function that can be inlined.") .onSet(fun v => void(SpcTuning.maxInlineParams = v)); - group.newIntOption("inline-max-depth", SpcTuning.maxInlineDepth, "Maximum inlining nesting depth.") + group.newIntOption("inline-max-depth", SpcTuning.maxInlineDepth, "Maximum inlining nesting depth for regular (non-fast) functions.") .onSet(fun v => void(SpcTuning.maxInlineDepth = v)); group.newIntOption("fast-inline-max-depth", SpcTuning.maxFastInlineDepth, "Maximum inlining nesting depth for fast functions.") .onSet(fun v => void(SpcTuning.maxFastInlineDepth = v)); diff --git a/src/engine/compiler/SinglePassCompiler.v3 b/src/engine/compiler/SinglePassCompiler.v3 index 073480f8a..37d5b4aee 100644 --- a/src/engine/compiler/SinglePassCompiler.v3 +++ b/src/engine/compiler/SinglePassCompiler.v3 @@ -60,6 +60,12 @@ component SpcConsts { def kindToFlags(kind: ValueKind) -> byte { return byte.view(kind.tag) << 4; } + + // {RegAlloc} assignment of an interpreter register a fast handler has not borrowed. Such + // a register holds nothing else; the register itself says which interpreter variable it + // is. Large enough that {forEachAssignmentInRange} skips it and that indexing the value + // stack with it fails loudly. + def IVAR_SLOT = 2000000000; } // Shorten constants inside this file. @@ -77,9 +83,11 @@ def KIND_V128 = SpcConsts.KIND_V128; def KIND_REF = SpcConsts.KIND_REF; def KIND_REF_U64 = SpcConsts.KIND_REF_U64; def KIND_CONT = SpcConsts.KIND_CONT; +def IVAR_SLOT = SpcConsts.IVAR_SLOT; +def BORROWABLE_IVARS = X86_64MasmRegs.BORROWABLE_IVARS; // Compiles Wasm bytecode to machine code in a single pass via a MacroAssembler. -class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAlloc, extensions: Extension.set, limits: Limits) extends BytecodeVisitor { +class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAlloc, extensions: Extension.set, limits: Limits, fast: bool) extends BytecodeVisitor { def instrTracer = if(Trace.compiler, InstrTracer.new()); def config = masm.regConfig; def regs = xenv; @@ -116,7 +124,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl var last_probe = 0; var skip_to_end: bool; var whamm_config: WhammInlineConfig; - var frames_reconstructed = false; + var fast_operand_size: int; // XXX: hack var handler_dest_info = Vector.new(); @@ -142,7 +150,8 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl return if(ok, osr_entry_label); } private def gen0(module: Module, func: FuncDecl) -> bool { - if (Trace.compiler) OUT.put1("==== begin compile: %q ========================", func.render(module.names, _)).ln(); + if (Trace.compiler) + OUT.put2("==== begin compile: %q %s==================", func.render(module.names, _), if(fast, "(fast)", "======")).ln(); var before_code_bytes = masm.curCodeBytes(); var before_data_bytes = masm.curDataBytes(); @@ -168,11 +177,16 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl // Push initial frame for top-level function state.frame_stack.clear(); + if (fast) { + // push an SpcFrame representing the calling interpreter + var interp_frame = SpcFrame.new(null, module, 0, 0, 0, -1, null); + pushSpcFrame(interp_frame); + } var initial_frame = SpcFrame.new(func, module, 0, 0, func.num_slots(), 0, masm.newLabel(func.cur_bytecode.length)); pushSpcFrame(initial_frame); // Emit prologue, which allocates the frame and initializes various registers. - emitPrologue(); + if (fast) emitFastPrologue(); else emitPrologue(); if (Trace.compiler && Trace.asm) traceAsm(); // Visit all local declarations. @@ -320,6 +334,8 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl } for (i = 1; i < config.regSet.length; i++) { var r = Reg(byte.view(i)); + // A register holding an IVar is allocated without occupying a value-stack slot. + if (fast && state.holdsIVar(r)) continue; var buf = StringBuilder.new().puts("{"); regAlloc.forEachAssignment(r, appendSlots(buf, _)); var slots = buf.puts("}").toString(); @@ -376,6 +392,23 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_mov_m_r(ValueKind.REF, frame.mem0_base_slot, regs.mem0_base); } } + def emitFastPrologue() { + // compute fast operand size to skip over + fast_operand_size = Opcodes.minimalUleb32Length(func.func_index); + // Advance IP past the fast-call opcode. + def r_ip = X86_64MasmRegs.INT_EXEC_ENV.ip; + masm.emit_addw_r_i(r_ip, fast_operand_size); // increment ip + masm.emit_compute_pc(-(fast_operand_size + 1)); // compute r_curpc + // save the registers computed above to the frame + masm.emit_save_ivar(r_ip); + masm.emit_save_ivar(X86_64MasmRegs.INT_EXEC_ENV.stp); + if (!FeatureDisable.stacktraces) masm.emit_save_ivar(X86_64MasmRegs.INT_EXEC_ENV.curpc); + allocIVars(); + + // Compute VFP = VSP - sig.params.length * SLOT_SIZE + masm.emit_mov_r_r(ValueKind.REF, regs.vfp, regs.vsp); // XXX: use 3-addr adjustment of VFP + masm.emit_subw_r_i(regs.vfp, sig.params.length * masm.valuerep.slot_size); + } def visitLocalDecl(count: u32, vtc: ValueTypeCode) { var vt = vtc.toAbstractValueType(module); var sp = state.sp; @@ -971,10 +1004,11 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl resolver.emitMoves(); state.sp = count; // adjust frame - masm.emit_addw_r_i(regs.sp, frame.frameSize); + if (!fast) masm.emit_addw_r_i(regs.sp, frame.frameSize); } def visitCallIndirect(op: Opcode, sig_index: u31, table_index: u31, tailCall: bool) { var sig = SigDecl.!(module.heaptypes[sig_index]); + if (fast) state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); withReconstructedInlinedFrames(fun { var vsp_reg = allocTmpFixed(ValueKind.REF, regs.vsp); var sv = popFixedReg(regs.func_arg); @@ -1005,7 +1039,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl // Non-final signatures do a fast-path check and then fallback to a runtime call. var sigokpt = masm.newLabel(it.pc); masm.emit_breq_r_i(tmp_reg, sig.canon_id, sigokpt); - state.emitSaveAll(resolver, SpillMode.SAVE_AND_REMEMBER_STORED); // XXX: no need to save all regs? + if (!fast) state.emitSaveAll(resolver, SpillMode.SAVE_AND_REMEMBER_STORED); // XXX: no need to save all regs? var arg2 = regs.runtime_arg2; // load from table into runtime arg0 masm.emit_v3_Table_funcs_r_r(arg2, table_reg); @@ -1041,12 +1075,13 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl setUnreachable(); return; } - var vsp_reg = allocTmpFixed(ValueKind.REF, regs.vsp); - sv = popFixedReg(regs.func_arg); - var tmp = allocTmp(ValueKind.REF); - var func_reg = sv.reg; - - emitCallToReg(sig, func_reg, vsp_reg, tmp, true, tailCall); + if (fast) state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); + withReconstructedInlinedFrames(fun { + var vsp_reg = allocTmpFixed(ValueKind.REF, regs.vsp); + var svf = popFixedReg(regs.func_arg); + var tmp = allocTmp(ValueKind.REF); + emitCallToReg(sig, svf.reg, vsp_reg, tmp, true, tailCall); + }); } def visit_DROP() { dropN(1); @@ -2013,7 +2048,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_call_runtime_op(op); }; // Reconstruct stack frames across runtime calls that might (Wasm-level) trap. - if (canTrap) withReconstructedInlinedFrames(emit); else emit(); + if (canTrap || fast) withReconstructedInlinedFrames(emit); else emit(); masm.emit_get_curstack(regs.scratch); masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch); dropN(args); @@ -2037,7 +2072,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_call_runtime_op(op); }; // Reconstruct stack frames across runtime calls that might (Wasm-level) trap. - if (canTrap) withReconstructedInlinedFrames(emit); else emit(); + if (canTrap || fast) withReconstructedInlinedFrames(emit); else emit(); masm.emit_get_curstack(regs.scratch); masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch); dropN(args); @@ -2124,16 +2159,25 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl } } - if (isInlined()) return; + if (!needsEpilogue()) return; // Compute VSP = VFP + state.sp emit_compute_vsp(regs.vsp, state.sp); - // Return to caller - masm.emit_mov_r_i(regs.ret_throw, 0); - // Deallocate stack frame - masm.emit_addw_r_i(regs.sp, frame.frameSize); - masm.emit_ret(); + if (!fast) { + // Return to caller + masm.emit_mov_r_i(regs.ret_throw, 0); + // Deallocate stack frame + masm.emit_addw_r_i(regs.sp, frame.frameSize); + masm.emit_ret(); + } else { + // restore vfp + masm.emit_mov_r_m(ValueKind.REF, regs.vfp, frame.vfp_slot); + // restore interpreter registers (but only if they were ever borrowed in this function) + for (reg in BORROWABLE_IVARS) restoreIVar(reg); + emitFastDispatch(); + } } + def emitFastDispatch() -> void; def emitOsrEntry(osr_entry_label: MasmLabel, state: Array) { if (Trace.compiler) Trace.OUT.put1(" OSR (+%d)", osr_entry_label.create_pos).ln(); masm.bindLabel(osr_entry_label); @@ -2153,7 +2197,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl def emitTrapReturn(label: MasmLabel, reason: TrapReason) { if (label != null) masm.bindLabel(label); masm.emit_mov_r_trap(regs.ret_throw, reason); - masm.emit_addw_r_i(regs.sp, frame.frameSize); + if (!fast) masm.emit_addw_r_i(regs.sp, frame.frameSize); masm.emit_ret(); } def newTrapLabel(reason: TrapReason) -> MasmLabel { @@ -2162,28 +2206,35 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl trap_labels.put((reason, label, frames)); return label; } - def getSpcInlinedFrameIp() -> long; + def getSpcInlinedFrameIp(outermost: bool) -> long; + // In fast mode the interpreter's registers are pre-allocated to {IVAR_SLOT}, so the + // allocator hands one out only by evicting it (at cost 50, with nothing to spill), and + // the epilogue reloads exactly the ones that were evicted. + def allocIVars() { + for (reg in BORROWABLE_IVARS) regAlloc.assign(reg, IVAR_SLOT); + } + def evictIVars() { + for (reg in BORROWABLE_IVARS) if (state.holdsIVar(reg)) regAlloc.free(reg); + } + // Reloads {reg} from the interpreter frame unless it still holds the interpreter's value. + def restoreIVar(reg: Reg) { + if (state.holdsIVar(reg)) return; + evictAndAssign(reg, IVAR_SLOT); + emitRestoreIVar(reg); + } + def emitRestoreIVar(reg: Reg); // Emit code to materialize stack frames for each inlined function. def emitReconstructStackFrames(frames: Array) -> int { + unrefRegs(); Metrics.spc_static_reconst.val++; masm.emit_inc_metric(Metrics.spc_dynamic_reconst); - def real_frame = frames[0]; - masm.emit_mov_m_i(xenv.pc_slot, real_frame.pc); - - // load instance - var inst_reg = allocTmp(ValueKind.REF); - masm.emit_mov_r_m(ValueKind.REF, inst_reg, frame.instance_slot); - var mem_reg = allocTmp(ValueKind.REF); - masm.emit_mov_r_m(ValueKind.REF, mem_reg, frame.mem0_base_slot); - // Load instance.functions - def func_reg = allocTmp(ValueKind.REF); - masm.emit_v3_Instance_functions_r_r(func_reg, inst_reg); - def vfp_reg = allocTmp(ValueKind.REF); - masm.emit_mov_r_m(ValueKind.REF, vfp_reg, frame.vfp_slot); - var prev_base_sp = int.view(frames[0].local_base_sp); - var wasm_func_reg = allocTmp(ValueKind.REF); + var inst_reg: Reg, mem_reg: Reg, func_reg: Reg, wasm_func_reg: Reg; + def vfp_reg: Reg = allocTmp(ValueKind.REF); var inl_inst_reg: Reg, inl_mem0_reg: Reg; + + var prev_base_sp = int.view(frames[0].local_base_sp); + if (whamm_config.is_inlined) { // XXX check individual configs? inl_inst_reg = allocTmp(ValueKind.REF); inl_mem0_reg = allocTmp(ValueKind.REF); @@ -2191,6 +2242,28 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_mov_r_m(ValueKind.REF, inl_mem0_reg, frame.inlined_mem0_base_slot); } + if (fast) { + masm.emit_mov_r_r(ValueKind.REF, vfp_reg, regs.vfp); + + inst_reg = regs.instance; // |\ use regs from interpreter + mem_reg = regs.mem0_base; // |/ + } else { + def real_frame = frames[0]; + masm.emit_mov_m_i(xenv.pc_slot, real_frame.pc); + + masm.emit_mov_r_m(ValueKind.REF, vfp_reg, frame.vfp_slot); + + inst_reg = allocTmp(ValueKind.REF); + masm.emit_mov_r_m(ValueKind.REF, inst_reg, frame.instance_slot); + mem_reg = allocTmp(ValueKind.REF); + masm.emit_mov_r_m(ValueKind.REF, mem_reg, frame.mem0_base_slot); + } + + // Load Instance.functions + func_reg = allocTmp(ValueKind.REF); + wasm_func_reg = allocTmp(ValueKind.REF); + masm.emit_v3_Instance_functions_r_r(func_reg, inst_reg); + // Pre-allocate stack space for all reconstructed frames at once. def total_space = (frames.length - 1) * (frame.frameSize + 8); masm.emit_subw_r_i(regs.sp, total_space); @@ -2200,18 +2273,16 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl def frame_info = frames[i]; def cur_base_sp = int.view(frame_info.local_base_sp); def delta = (cur_base_sp - prev_base_sp) * masm.valuerep.slot_size; - emitReconstructStackFrame(frame_info, frames.length - i - 1, delta, + emitReconstructStackFrame(frame_info, frames.length - i - 1, delta, getSpcInlinedFrameIp(i == 1), wasm_func_reg, func_reg, inst_reg, mem_reg, vfp_reg, inl_inst_reg, inl_mem0_reg); prev_base_sp = cur_base_sp; } + unrefRegs(); return total_space; } - def emitReconstructStackFrame(spcFrame: SpcFrame, offset: int, vfp_delta: int, + def emitReconstructStackFrame(spcFrame: SpcFrame, offset: int, vfp_delta: int, return_addr: long, wasm_func_reg: Reg, func_reg: Reg, inst_reg: Reg, mem_reg: Reg, vfp_reg: Reg, inl_inst_reg: Reg, inl_mem0_reg: Reg) { - // Use inlined frame stub IP as return address for all reconstructed frames - def return_addr = getSpcInlinedFrameIp(); - def frame_offset = offset * (frame.frameSize + 8); // Write inlined frame stub IP as return address def retaddr_slot = MasmAddr(regs.sp, frame_offset + frame.frameSize); @@ -2259,27 +2330,21 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl // Guards compiler code with frame reconstruction (if necessary). def withReconstructedInlinedFrames(emit: void -> void) { if (isInlined()) { - if (frames_reconstructed) { - // FIXME this should not happen (but does): - // - in the case of deep nesting when one layer is a Whamm probe - // - when refactoring to avoid `with` clause, GC test fails (inlining depth 2) - if (Trace.compiler) Trace.OUT.puts(" nested frame reconstruction inhibited\n"); - emit(); - return; - } - unrefRegs(); - frames_reconstructed = true; def space = emitReconstructStackFrames(snapshotFrames()); emit(); - frames_reconstructed = false; if (space > 0) { - masm.emit_addw_r_i(regs.sp, space); - masm.emit_mov_r_m(ValueKind.REF, regs.vfp, frame.vfp_slot); + if (fast) { + // reload VFP from the deepest inlined frame + masm.emit_mov_r_m(ValueKind.REF, regs.vfp, frame.vfp_slot.plus(space - frame.frameSize - 8)); + masm.emit_addw_r_i(regs.sp, space); + } else { + masm.emit_addw_r_i(regs.sp, space); + masm.emit_mov_r_m(ValueKind.REF, regs.vfp, frame.vfp_slot); + } } } else { emit(); } - } def unsupported() { success = false; // XXX: add opcode @@ -2372,8 +2437,13 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_mov_m_r(ValueKind.REF, frame.vsp_slot, reg); } def emit_reload_regs() { - // XXX: recompute VFP from VSP - #slots? - masm.emit_mov_r_m(ValueKind.REF, regs.vfp, frame.vfp_slot); + if (fast) { + emit_load_instance(regs.instance); + evictIVars(); // an outcall may have clobbered any of them; the epilogue reloads + } else { + // XXX: recompute VFP from VSP - #slots? + masm.emit_mov_r_m(ValueKind.REF, regs.vfp, frame.vfp_slot); + } if (module.memories.length > 0) { if (whamm_config.is_inlined) { masm.emit_mov_r_m(ValueKind.REF, regs.mem0_base, frame.inlined_mem0_base_slot); @@ -2503,10 +2573,14 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl return reg; } def allocFixed(kind: ValueKind, reg: Reg, slot: u32) -> Reg { - if (!regAlloc.isFree(reg)) spillRegAndFree(reg); - regAlloc.assign(reg, int.!(slot)); + evictAndAssign(reg, int.!(slot)); return reg; } + // Evicts whatever {reg} currently holds (if anything) and assigns it to {slot}. + def evictAndAssign(reg: Reg, slot: int) { + if (!regAlloc.isFree(reg)) spillRegAndFree(reg); + regAlloc.assign(reg, slot); + } def ensureReg(sv: SpcVal, slot: u32) -> Reg { var reg = sv.reg; if (reg == NO_REG) { @@ -2541,6 +2615,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl return regAlloc.alloc(kind, slot); } def addSpillCost(score: int, reg: Reg, slot: int) -> int { + if (slot == IVAR_SLOT) return 50 + score; // borrowable IVar; must precede the tmp-slot test if (slot >= state.sp) return 100000000 + score; // tmp slot? if (!state.state[slot].isStored()) score += 100; // will generate a spill if (slot < 10 && slot < func.num_locals) score += 10; // penalize first 10 locals @@ -2667,6 +2742,8 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl } def emitSpill(slot: int) { if (slot == TMP_SLOT) return; + // IVars are either saved in the prologue or already correct in the frame. + if (slot == IVAR_SLOT) return; var sv = state.state[slot]; if (sv.isConst()) return void(state.state[slot] = sv.withoutReg()); if (sv.isStored()) return void(state.state[slot] = sv.withoutReg()); @@ -2717,10 +2794,10 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl } state.frame_stack.push(frame); // Update cached copies from new top frame - it.reset(frame.func).at(frame.pc, -1); + if (frame.func != null) it.reset(frame.func).at(frame.pc, -1); // null = fast: interpreted caller module = frame.module; func = frame.func; - sig = func.sig; + sig = if(func != null, func.sig); num_locals = frame.num_locals; local_base_sp = frame.local_base_sp; ctl_base_sp = frame.ctl_base_sp; @@ -2746,8 +2823,12 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl def isInlined() -> bool { return state.frame_stack.top > 1; } + def needsEpilogue() -> bool { + // inlined callees will fallthrough and don't need epilogue to be emitted + return !isInlined() || ctl_base_sp == 0; + } def inlineDepth() -> int { - return state.frame_stack.top - 1; + return if(fast, state.frame_stack.top - 2, state.frame_stack.top - 1); } def snapshotFrames() -> Array { var frames = Array.new(state.frame_stack.top); @@ -2763,7 +2844,8 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl if (Trace.compiler) OUT.put1(" inline func #%d? ", func.func_index); if (func.imp != null) return no("imported"); - if (inlineDepth() >= SpcTuning.maxInlineDepth) return no("max inline depth exceeded"); + var maxDepth = if(fast, SpcTuning.maxFastInlineDepth, SpcTuning.maxInlineDepth); + if (inlineDepth() >= maxDepth) return no("max inline depth exceeded"); if (func.orig_bytecode.length > SpcTuning.maxInlineBytecodeSize) return no("func too large"); if (func.sig.params.length > SpcTuning.maxInlineParams) return no("too many parameters"); @@ -2954,6 +3036,21 @@ class SpcState(regAlloc: RegAlloc) { var ctl_stack = ArrayStack.new(); // Frame management (for inlining) var frame_stack = ArrayStack.new(); + // Frees every register except those holding an interpreter variable ({IVAR_SLOT}): a + // fast handler pre-allocates them, and a reset must not lose that. + private def clearExceptIVars() { + regAlloc.forEachAllocatedReg(NO_REG, freeUnlessIVar); + } + private def freeUnlessIVar(reg: Reg) { + if (!holdsIVar(reg)) regAlloc.free(reg); + } + def holdsIVar(reg: Reg) -> bool { + found_ivar = false; + regAlloc.forEachAssignment(reg, noteIVar); + return found_ivar; + } + private var found_ivar: bool; + private def noteIVar(slot: int) { if (slot == IVAR_SLOT) found_ivar = true; } // Reset the state for starting a new function. def reset(sig: SigDecl, ret_label: MasmLabel) { @@ -3050,7 +3147,7 @@ class SpcState(regAlloc: RegAlloc) { resetTo(max, ctl.merge_state); } else { // merge not reached; push "bottom" value for all results - regAlloc.clear(); + clearExceptIVars(); sp = ctl.val_stack_top; for (i < sp) { // XXX: we clear register allocation state mostly for debug. var sv = state[i]; @@ -3129,10 +3226,14 @@ class SpcState(regAlloc: RegAlloc) { return SpcVal((from.flags & ~(IS_CONST)) | force_store, from.reg, 0); } private def resetTo(max: u32, nstate: Array) { - regAlloc.clear(); + clearExceptIVars(); for (i < max) { var sv = nstate[i]; - if (sv.inReg()) regAlloc.assign(sv.reg, int.!(i)); + if (sv.inReg()) { + // the merge's move clobbers any interpreter variable still in {sv.reg} + if (holdsIVar(sv.reg)) regAlloc.free(sv.reg); + regAlloc.assign(sv.reg, int.!(i)); + } state[i] = sv; } sp = max; diff --git a/src/engine/v3/V3Interpreter.v3 b/src/engine/v3/V3Interpreter.v3 index 09abc5e90..5556138b0 100644 --- a/src/engine/v3/V3Interpreter.v3 +++ b/src/engine/v3/V3Interpreter.v3 @@ -371,7 +371,12 @@ class V3Interpreter extends WasmStack { RETURN => { doReturn(frame.fp, frame.func.sig); } - CALL => { + CALL, + FAST_CALL0, FAST_CALL1, FAST_CALL2, FAST_CALL3, FAST_CALL4, FAST_CALL5, FAST_CALL6, FAST_CALL7, + FAST_CALL8, FAST_CALL9, FAST_CALL10, FAST_CALL11, FAST_CALL12, FAST_CALL13, FAST_CALL14, FAST_CALL15, + FAST_CALL16, FAST_CALL17, FAST_CALL18, FAST_CALL19, FAST_CALL20, FAST_CALL21, FAST_CALL22, FAST_CALL23, + FAST_CALL24, FAST_CALL25, FAST_CALL26, FAST_CALL27, FAST_CALL28, FAST_CALL29, FAST_CALL30, FAST_CALL31, + FAST_CALL32, FAST_CALL33, FAST_CALL34, FAST_CALL35, FAST_CALL36, FAST_CALL37, FAST_CALL38, FAST_CALL39 => { var func_index = codeptr.read_uleb32(); var f = frame.func.instance.functions[func_index]; return doCallFunction(f); @@ -1651,7 +1656,12 @@ class V3Interpreter extends WasmStack { // XXX: use read_opcode_and_skip() var opcode = codeptr.read_opcode_but_skip_probe(frame.func.decl); match (opcode) { - CALL, CALL_REF => { + CALL, CALL_REF, + FAST_CALL0, FAST_CALL1, FAST_CALL2, FAST_CALL3, FAST_CALL4, FAST_CALL5, FAST_CALL6, FAST_CALL7, + FAST_CALL8, FAST_CALL9, FAST_CALL10, FAST_CALL11, FAST_CALL12, FAST_CALL13, FAST_CALL14, FAST_CALL15, + FAST_CALL16, FAST_CALL17, FAST_CALL18, FAST_CALL19, FAST_CALL20, FAST_CALL21, FAST_CALL22, FAST_CALL23, + FAST_CALL24, FAST_CALL25, FAST_CALL26, FAST_CALL27, FAST_CALL28, FAST_CALL29, FAST_CALL30, FAST_CALL31, + FAST_CALL32, FAST_CALL33, FAST_CALL34, FAST_CALL35, FAST_CALL36, FAST_CALL37, FAST_CALL38, FAST_CALL39 => { codeptr.skip_leb(); frame.pc = codeptr.pos; } diff --git a/src/engine/x86-64/V3Offsets.v3 b/src/engine/x86-64/V3Offsets.v3 index 9fb90e5d5..797722ccc 100644 --- a/src/engine/x86-64/V3Offsets.v3 +++ b/src/engine/x86-64/V3Offsets.v3 @@ -30,6 +30,7 @@ class V3Offsets { def FuncDecl_orig_bytecode = int.view(Pointer.atField(decl.orig_bytecode) - Pointer.atObject(decl)); def FuncDecl_sidetable = int.view(Pointer.atField(decl.sidetable.entries) - Pointer.atObject(decl)); def FuncDecl_target_code = int.view(Pointer.atField(decl.target_code.spc_entry) - Pointer.atObject(decl)); + def FuncDecl_fast_target_code = int.view(Pointer.atField(decl.fast_target_code.spc_entry) - Pointer.atObject(decl)); def FuncDecl_tierup_trigger = int.view(Pointer.atField(decl.tierup_trigger) - Pointer.atObject(decl)); def FuncDecl_entry_probed = int.view(Pointer.atField(decl.entry_probed) - Pointer.atObject(decl)); def FuncDecl_frame_var_tags = int.view(Pointer.atField(decl.frame_var_tags) - Pointer.atObject(decl)); diff --git a/src/engine/x86-64/X86_64Interpreter.v3 b/src/engine/x86-64/X86_64Interpreter.v3 index 34563cf64..b000c362b 100644 --- a/src/engine/x86-64/X86_64Interpreter.v3 +++ b/src/engine/x86-64/X86_64Interpreter.v3 @@ -530,7 +530,7 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { // Spill VSP (value stack pointer) asm.movq_m_r(m_vsp, r_v3_vsp); // load dispatch table into register - if (!FeatureDisable.globalProbes) masm.emit_load_dispatch_table_reg(xenv.dispatch); + masm.emit_load_dispatch_table_reg(); // move WasmFunction into tmp asm.movq_r_r(tmp, r_v3_wasm_func); masm.emit_restore_ivar(xenv.vsp); @@ -547,7 +547,7 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { // Spill VSP (value stack pointer) asm.movq_m_r(m_vsp, r_vsp); // load dispatch table into register - if (!FeatureDisable.globalProbes) masm.emit_load_dispatch_table_reg(xenv.dispatch); + masm.emit_load_dispatch_table_reg(); // move WasmFunction into tmp asm.movq_r_r(tmp, r_func_arg); asm.jmp_rel_near(shared_entry); @@ -1306,6 +1306,13 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { asm.movd_r_i(G(Target.V3_RET_GPRS[0]), 0); genPopFrameAndRet(); + // FAST_CALL + // these are placeholders in the dispatch table, will be patched with real handler SPC code + if (FastIntTuning.useFastFunctions) { + for (op in Opcodes.fast_calls) bindHandler(op); + masm.emit_intentional_crash(); + } + bindHandler(Opcode.CALL); computeCurIpForTrap(-1); genReadUleb32(r_tmp1); diff --git a/src/engine/x86-64/X86_64MacroAssembler.v3 b/src/engine/x86-64/X86_64MacroAssembler.v3 index 9e341f030..31c4a16f3 100644 --- a/src/engine/x86-64/X86_64MacroAssembler.v3 +++ b/src/engine/x86-64/X86_64MacroAssembler.v3 @@ -992,9 +992,9 @@ class X86_64MacroAssembler extends MacroAssembler { def emit_restore_ivar(reg: Reg) { for (t in INT_EXEC_ENV.all_ivars) if (t.0 == reg) { asm.movq_r_m(G(reg), R.RSP.plus(t.1)); return; } } - def emit_load_dispatch_table_reg(reg: Reg) { - var offsets = getOffsets(); - asm.movq_r_m(G(reg), absPointer(offsets.Interpreter_dispatchTable)); + def emit_load_dispatch_table_reg() { + if (FeatureDisable.globalProbes) return; + asm.movq_r_m(G(INT_EXEC_ENV.dispatch), absPointer(getOffsets().Interpreter_dispatchTable)); } def emit_restore_dispatch_table_reg(reg: Reg) { asm.movq_r_m(G(reg), absPointer(getOffsets().Interpreter_dispatchTable)); @@ -1609,6 +1609,24 @@ class X86_64MacroAssembler extends MacroAssembler { def emit_u64_from_ref_u64(to: Reg, from: Reg) { asm.pextrq_r_s_i(G(to), X(from), 1); } + def emit_int_dispatch(opcode: X86_64Gpr, base: X86_64Gpr, r_ip: X86_64Gpr, r_dispatch: X86_64Gpr, + ptr: X86_64Addr, table: IcCodeRef, increment: bool, ic: X86_64InterpreterCode) { + if (ptr != null) asm.movbzx_r_m(opcode, ptr); + if (increment) asm.inc_r(r_ip); + match (FastIntTuning.dispatchEntrySize) { + 4 => { + if (table == null) { + asm.movd_r_m(base, r_dispatch.plusR(opcode, 4, 0)); + } else { + var addr = ic.start + table.offset; + asm.movd_r_m(base, X86_64Addr.new(null, opcode, 4, int.!(addr - Pointer.NULL))); + } + asm.ijmp_r(base); + } + _ => System.error("X86_64MacroAssembler", + Strings.format1("emit_int_dispatch called on unsupported dispatch entry size: %d", FastIntTuning.dispatchEntrySize)); + } + } // Reads a 32- or 64-bit unsigned LEB from {rw_ptr} into {w_dest}. def emit_read_uleb(w_dest: X86_64Gpr, rw_ptr: X86_64Gpr, w_scratch1: X86_64Gpr, w_scratch2: X86_64Gpr) -> this { diff --git a/src/engine/x86-64/X86_64MasmRegs.v3 b/src/engine/x86-64/X86_64MasmRegs.v3 index 9481c1bf5..e1466731a 100644 --- a/src/engine/x86-64/X86_64MasmRegs.v3 +++ b/src/engine/x86-64/X86_64MasmRegs.v3 @@ -167,6 +167,10 @@ component X86_64MasmRegs { def SPC_EXEC_ENV = t.0; // The execution environment for interpreter compilation contexts. def INT_EXEC_ENV = t.1; + // Interpreter registers a fast handler may borrow. The first three must be saved to the + // frame before use (the register is the only live copy); the rest are restore-only. + def BORROWABLE_IVARS: Array = [INT_EXEC_ENV.ip, INT_EXEC_ENV.stp, INT_EXEC_ENV.curpc, + INT_EXEC_ENV.func_decl, INT_EXEC_ENV.eip, INT_EXEC_ENV.dispatch]; // A register allocator for single-pass compilation contexts. def SPC_ALLOC = (fun -> RegAlloc { @@ -186,6 +190,17 @@ component X86_64MasmRegs { return RegAlloc.new(CONFIG.poolMap, pools, null); })(); + // A register allocator for fast handlers in the SPC. + def FAST_INT_ALLOC = (fun -> RegAlloc { + var pools = [ + // Restore-only IVars precede save-required ones: all IVars tie on spill cost, and + // ties evict by pool order. + RegPool32.new([RCX, RDX, R8, R9, R12, R13, R14, RAX, RBX, R15]), + RegPool32.new([XMM2, XMM3, XMM4, XMM5, XMM6, XMM7, XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14]) + ]; + return RegAlloc.new(CONFIG.poolMap, pools, null); + })(); + def toGpr(reg: Reg) => GPRS[reg.index]; def toXmmr(reg: Reg) => XMMS[reg.index]; diff --git a/src/engine/x86-64/X86_64SinglePassCompiler.v3 b/src/engine/x86-64/X86_64SinglePassCompiler.v3 index eb56cf68d..ed433d358 100644 --- a/src/engine/x86-64/X86_64SinglePassCompiler.v3 +++ b/src/engine/x86-64/X86_64SinglePassCompiler.v3 @@ -23,17 +23,41 @@ def KIND_F64 = SpcConsts.KIND_F64; def KIND_V128 = SpcConsts.KIND_V128; def KIND_REF = SpcConsts.KIND_REF; +def xenv: IntExecEnv = X86_64MasmRegs.INT_EXEC_ENV; + +def r_stp = G(xenv.stp); +def r_ip = G(xenv.ip); +def r_eip = G(xenv.eip); +def r_func_decl = G(xenv.func_decl); +def r_curpc = G(xenv.curpc); +def ip_ptr = r_ip.plus(0); +def r_dispatch = G(xenv.dispatch); +def r_tmp0 = G(xenv.tmp0); // RCX +def r_tmp1 = G(xenv.tmp1); // RDX + +def m_code = R.RSP.plus(X86_64InterpreterFrame.code.offset); + // Implements the target-specific parts of the single-pass compiler for X86-64. class X86_64SinglePassCompiler extends SinglePassCompiler { def w = DataWriter.new(); def mmasm = X86_64MacroAssembler.new(w, X86_64MasmRegs.CONFIG); def asm = mmasm.asm; + var ic: X86_64InterpreterCode; - new(extensions: Extension.set, limits: Limits, config: RegConfig) - super(X86_64MasmRegs.SPC_EXEC_ENV, mmasm, X86_64MasmRegs.SPC_ALLOC.copy(), extensions, limits) { + new(ic, extensions: Extension.set, limits: Limits, config: RegConfig, fast: bool) + super(X86_64MasmRegs.SPC_EXEC_ENV, mmasm, + if(fast, X86_64MasmRegs.FAST_INT_ALLOC.copy(), X86_64MasmRegs.SPC_ALLOC.copy()), + extensions, limits, fast) { mmasm.trap_stubs = TRAPS_STUB; } - + def emitFastDispatch() { + mmasm.emit_int_dispatch(r_tmp0, r_tmp1, r_ip, r_dispatch, ip_ptr, + if(ic != null, IcCodeRef.new(ic.header.fastDispatchTableOffset)), true, ic); + } + def emitRestoreIVar(reg: Reg) { + if (reg == X86_64MasmRegs.INT_EXEC_ENV.dispatch) mmasm.emit_load_dispatch_table_reg(); + else mmasm.emit_restore_ivar(reg); + } private def visitCompareI(asm: X86_64Assembler, cond: X86_64Cond) -> bool { var b = pop(), a = popReg(); if (b.isConst()) asm.cmp_r_i(G(a.reg), b.const); @@ -1172,8 +1196,9 @@ class X86_64SinglePassCompiler extends SinglePassCompiler { state.push(b.kindFlagsMatching(kind, IN_REG), b.reg, 0); return true; } - def getSpcInlinedFrameIp() -> long { - return long.view(INLINED_FRAME_STUB.start); + def getSpcInlinedFrameIp(outermost: bool) -> long { + if (fast && outermost) return long.view(X86_64PreGenStubs.getSpcIntEntry()); + return long.view(INLINED_FRAME_STUB.start + 1); } } @@ -1231,11 +1256,12 @@ class X86_64SpcCode extends RiUserCode { // Represents the JITed code for an entire module. class X86_64SpcModuleCode extends X86_64SpcCode { def mapping: Mapping; + def fast: bool; // true if this unit was compiled in fast mode var codeEnd: int; // for dynamically adding code to the end var sourcePcs: Vector<(int, List)>; var embeddedRefOffsets: Vector; - new(mapping) super("spc-module", mapping.range.start, mapping.range.end) { + new(mapping, fast) super("spc-module", mapping.range.start, mapping.range.end) { RiGc.registerScanner(this, X86_64SpcModuleCode.scan); } @@ -1287,11 +1313,22 @@ class X86_64SpcModuleCode extends X86_64SpcCode { var h = X86_64FrameHandle.Spc(p_rsp); if (inline_ctx == null) { h.set_curpc(-1); - } else if (inline_ctx.tail == null) { - h.set_curpc(inline_ctx.head.pc); } else { - p_rsp = reconstructInlinedFramesForTrap(p_rsp, inline_ctx); - (ucontext + ucontext_rsp_offset).store(p_rsp); + // A fast-compiled unit always reuses the calling interpreter's own + // frame (it never allocates its own), so that frame's function is + // never represented in the static inline_ctx for this unit; add it. + if (fast) { + def frame_wf = (p_rsp + X86_64InterpreterFrame.wasm_func.offset).load(); + def frame_pc = X86_64Interpreter.computePCFromFrame(p_rsp); + inline_ctx = Lists.fromArray(Arrays.append(FuncLoc(frame_wf.decl.func_index, frame_pc), Lists.toArray(inline_ctx))); + } + // Check on the size of the inline context + if (inline_ctx.tail == null) { + h.set_curpc(inline_ctx.head.pc); + } else { + p_rsp = reconstructInlinedFramesForTrap(p_rsp, inline_ctx); + (ucontext + ucontext_rsp_offset).store(p_rsp); + } } } @@ -1317,7 +1354,7 @@ class X86_64SpcModuleCode extends X86_64SpcCode { var pc = inlined[i].pc; r_rsp += -8; - r_rsp.store(INLINED_FRAME_STUB.start); + r_rsp.store(if(fast && i == count - 1, X86_64PreGenStubs.getSpcIntEntry(), INLINED_FRAME_STUB.start + 1)); r_rsp += -X86_64InterpreterFrame.size; // move rsp? var h = X86_64FrameHandle.Spc(r_rsp); @@ -1560,6 +1597,7 @@ def codePointer(f: P -> R) -> Pointer { // Global functionality associated with the single-pass compiler for X86-64. component X86_64Spc { + var ic: X86_64InterpreterCode; // A handy chokepoint for entering JIT code from V3. def invoke(wf: WasmFunction, sp: Pointer) -> Throwable { return V3_SPC_ENTRY_FUNC.get()(wf, sp, wf.decl.target_code.spc_entry); diff --git a/src/engine/x86-64/X86_64Target.v3 b/src/engine/x86-64/X86_64Target.v3 index 5ba77391f..75256d0b2 100644 --- a/src/engine/x86-64/X86_64Target.v3 +++ b/src/engine/x86-64/X86_64Target.v3 @@ -63,6 +63,43 @@ component Target { f.target_code = TargetCode(addr); Debug.afterCompile(f, u64.view(addr - Pointer.NULL)); } + def setFastTargetCode(f: FuncDecl, addr: Pointer, end: Pointer) { + if (Trace.compiler) { + Trace.OUT.put2("func[%d].fast_target_code: break *0x%x", f.func_index, addr - Pointer.NULL) + .put2(" disass 0x%x, 0x%x", addr - Pointer.NULL, end - Pointer.NULL).ln(); + if (Trace.asm) { + var cur_byte = addr; + Trace.OUT.puts("JIT code: "); + while (cur_byte < end) { + Trace.OUT.put1("%x ", cur_byte.load()); + cur_byte++; + } + Trace.OUT.ln(); + } + } + f.fast_target_code = TargetCode(addr); + patchFastCallDispatch(f, addr); + Debug.afterCompile(f, u64.view(addr - Pointer.NULL)); + } + def patchFastCallDispatch(f: FuncDecl, addr: Pointer) { + if (f.fast_call_idx < 0) return; + def opcode = Opcodes.indexToFastCall(f.fast_call_idx); + def ic = X86_64PreGenStubs.getInterpreterCode(); + // XXX Patching only fast (non-probed?) dispatch tables + def fast_offset = ic.header.fastDispatchTableOffset; + def entry = ic.start + fast_offset + opcode.code * FastIntTuning.dispatchEntrySize; + if (Trace.compiler) { + Trace.OUT.puts("patching dispatch table\n"); + Trace.OUT.put1(" start 0x%x\n", u64.view(ic.start)); + Trace.OUT.put1(" entry 0x%x\n", u64.view(entry)); + Trace.OUT.put1(" addr 0x%x\n", u64.view(addr)); + } + match (FastIntTuning.dispatchEntrySize) { + 4 => entry.store(u32.view(addr)); + _ => System.error("patchFastCallDispatch", "unsupported dispatchEntrySize"); + } + } + def pregenIntoFile(filename: string) -> ErrorBuilder { var data = System.fileLoad(filename); var err = ErrorBuilder.new().puts("interpreter generator: "); @@ -194,6 +231,12 @@ class TargetHandlerDest(is_dummy: bool) { } class X86_64ExecutionStrategy extends ExecutionStrategy { + var hasMonitors = false; + + // Called if monitors will be attached to the (forthcoming) module. + def onMonitorsStart() { + hasMonitors = true; + } // Call a function with arguments and return a result. def call(func: Function, args: Range) -> Result { return X86_64StackManager.runOnFreshStack(func, args); // XXX: specialize for different strategies? @@ -208,6 +251,102 @@ class X86_64ExecutionStrategy extends ExecutionStrategy { def disableLazyNameDecodingDuringGC(module: Module) { if (module.names != null) module.names.lazyDecodeDisabled = RiGc.inGC; } + + def installStubForModule(module: Module, set: (Module, FuncDecl) -> void) { + // ensure entrypoint and lazy compile stubs are generated + X86_64PreGenStubs.gen(); + // Set all functions to refer to the tier-up compile stub. + var codeSize = MINIMUM_CODE_SIZE; + for (i < module.functions.length) { + var f = module.functions[i]; + if (f.imported()) continue; + set(module, f); + codeSize += X86_64Spc.estimateCodeSizeFor(f); + } + allocateCodeForModule(module, codeSize); + } + + def fastCompileEntireModule(module: Module, size: u32, err: ErrorGen, ballast: u32) { + // ensure entrypoint and lazy compile stubs are generated + X86_64PreGenStubs.gen(); + + var compiler = newCompiler(module.filename, true, null); + var w = compiler.w; + + // generate code for all fast functions + var bounds = Array<(int, int)>.new(module.functions.length); + for (i = 0; err.ok() && i < module.functions.length; i++) { + var f = module.functions[i]; + if (f.fast_call_idx < 0) continue; + var start = w.atEnd().pos; + var compiled = compiler.gen(module, f, err); + if (compiled) bounds[i] = (start, w.end()); + else bounds[i] = (-1, -1); + } + + // copy and map code (reserve32 ensures address fits in 32 bits for dispatch table patching) + var length = u64.view(w.atEnd().pos) + ballast; + var mapping = Mmap.reserve32(length, Mmap.PROT_WRITE), range = mapping.range; // TODO: handle failure + var masm = X86_64MacroAssembler.!(compiler.masm); + masm.setTargetAddress(u64.view(range.start - Pointer.NULL)); + Target.copyInto(mapping.range, 0, w); + // TODO: for security, move embedded references out of the code region and make it non-writable + Mmap.protect(range.start, u64.!(range.end - range.start), Mmap.PROT_WRITE | Mmap.PROT_READ | Mmap.PROT_EXEC); + // Briefly relax the dispatch table to writable, then restore it below. + var ic = X86_64PreGenStubs.getInterpreterCode(); + var dispatchRegionSize = u64.!(ic.header.codeEnd - ic.header.codeStart); + Mmap.protect(ic.start + ic.header.codeStart, dispatchRegionSize, + Mmap.PROT_READ | Mmap.PROT_WRITE | Mmap.PROT_EXEC); + for (i < bounds.length) { + var b = bounds[i]; + if (b.0 >= 0) { + var addr = mapping.range.start; + var f = module.functions[i]; + Target.setFastTargetCode(f, addr + b.0, addr + b.1); + } else { + var f = module.functions[i]; + if (Trace.compiler) Trace.OUT.put1("func[%d] initial compile failed", f.func_index).ln(); + var addr = X86_64Spc.setInterpreterFallback(f); + Target.patchFastCallDispatch(f, addr); + } + } + Mmap.protect(ic.start + ic.header.codeStart, dispatchRegionSize, + Mmap.PROT_READ | Mmap.PROT_EXEC); + // XXX: reduce duplication with {X86_64SpcModuleCode.appendCode}. + var code = X86_64SpcModuleCode.new(mapping, true); + if (masm.source_locs != null) { + code.sourcePcs = Vector.new(); + code.sourcePcs.putv(masm.source_locs); + } + if (masm.embeddedRefOffsets != null) { + if (code.embeddedRefOffsets == null) code.embeddedRefOffsets = Vector.new(); + code.embeddedRefOffsets.putv(masm.embeddedRefOffsets); + } + + module.target_module = TargetModule(code); + RiRuntime.registerUserCode(code); + module.target_module.spc_code.keepAlive(); + Debug.afterCompileModule(module); + // print out patched dispatch table + if (Trace.compiler) { + Trace.OUT.puts("Patched fast handlers:\n"); + for (i < module.fast_funcs.length) { + def func: FuncDecl = module.fast_funcs[i]; + var name: string; + for (j < module.exports.length) { + def exp = module.exports[j]; + if (exp.1 == func) { + name = exp.0; + break; + } + } + Trace.OUT.put2(" FAST_CALL%d: %s (", func.fast_call_idx, name); + func.render(module.names, Trace.OUT); + Trace.OUT.puts(")"); + Trace.OUT.ln(); + } + } + } } // One tier: fast-int, modules require no pre-processing. @@ -218,6 +357,12 @@ class X86_64InterpreterOnlyStrategy extends X86_64ExecutionStrategy { def onModuleFinish(module: Module, size: u32, err: ErrorGen) { disableLazyNameDecodingDuringGC(module); + // defer compilation until after monitors have installed probes + if (!hasMonitors && FastIntTuning.useFastFunctions) fastCompileEntireModule(module, size, err, 1024); + } + // Called after monitors have processed a module. + def onMonitorsFinish(module: Module, err: ErrorGen) { + if (FastIntTuning.useFastFunctions) fastCompileEntireModule(module, 0, err, 1024); } def onFuncValidationFinish(module: Module, func: FuncDecl, err: ErrorGen) { if (err != null && !err.ok()) return; @@ -260,7 +405,7 @@ class X86_64SpcStrategy extends X86_64ExecutionStrategy { allocateCodeForModule(module, MINIMUM_CODE_SIZE + X86_64Spc.estimateCodeSizeFor(wf.decl)); } var code = module.target_module.spc_code; - var compiler = newCompiler(module.filename); // XXX: cache per-thread + var compiler = newCompiler(module.filename, false, null); // XXX: cache per-thread var masm = X86_64MacroAssembler.!(compiler.masm), w = masm.asm.w; // generate code for the function @@ -289,37 +434,22 @@ class X86_64SpcStrategy extends X86_64ExecutionStrategy { } return SpcResultForStub(wf, entrypoint, null); } - def installStubForModule(module: Module, set: (Module, FuncDecl) -> void) { - // ensure entrypoint and lazy compile stubs are generated - X86_64PreGenStubs.gen(); - // Set all functions to refer to the tier-up compile stub. - var codeSize = MINIMUM_CODE_SIZE; - for (i < module.functions.length) { - var f = module.functions[i]; - if (f.imported()) continue; - set(module, f); - codeSize += X86_64Spc.estimateCodeSizeFor(f); - } - allocateCodeForModule(module, codeSize); - } } // One tier: SPC, modules are eagerly compiled. class X86_64SpcAotStrategy(interpreter_fallback: bool) extends X86_64SpcStrategy { - var hasMonitors = false; - - // Called if monitors will be attached to the (forthcoming) module. - def onMonitorsStart() { - hasMonitors = true; - } // Called after a module is parsed. def onModuleFinish(module: Module, size: u32, err: ErrorGen) { // defer compilation for AOT mode until after monitors have been installed - if (!hasMonitors) compileEntireModule(module, size, interpreter_fallback, err, 1024); + if (!hasMonitors) { + if (FastIntTuning.useFastFunctions) fastCompileEntireModule(module, size, err, 1024); + compileEntireModule(module, size, interpreter_fallback, err, 1024); + } disableLazyNameDecodingDuringGC(module); } // Called after monitors have processed a module. def onMonitorsFinish(module: Module, err: ErrorGen) { + if (FastIntTuning.useFastFunctions) fastCompileEntireModule(module, 0, err, 1024); compileEntireModule(module, 0, interpreter_fallback, err, 1024); } // Called before a test function is run. @@ -337,7 +467,7 @@ class X86_64SpcAotStrategy(interpreter_fallback: bool) extends X86_64SpcStrategy // ensure entrypoint and lazy compile stubs are generated X86_64PreGenStubs.gen(); - var compiler = newCompiler(module.filename); + var compiler = newCompiler(module.filename, false, null); var w = compiler.w; // generate code for all functions @@ -373,7 +503,7 @@ class X86_64SpcAotStrategy(interpreter_fallback: bool) extends X86_64SpcStrategy } } // XXX: reduce duplication with {X86_64SpcModuleCode.appendCode}. - var code = X86_64SpcModuleCode.new(mapping); + var code = X86_64SpcModuleCode.new(mapping, false); if (masm.source_locs != null) { code.sourcePcs = Vector.new(); code.sourcePcs.putv(masm.source_locs); @@ -394,9 +524,13 @@ class X86_64SpcAotStrategy(interpreter_fallback: bool) extends X86_64SpcStrategy class X86_64SpcLazyStrategy extends X86_64SpcStrategy { // Called after a module is parsed. def onModuleFinish(module: Module, size: u32, err: ErrorGen) { + if (!hasMonitors && FastIntTuning.useFastFunctions) fastCompileEntireModule(module, size, err, 1024); installStubForModule(module, X86_64Spc.setLazyCompileFor); disableLazyNameDecodingDuringGC(module); } + def onMonitorsFinish(module: Module, err: ErrorGen) { + if (FastIntTuning.useFastFunctions) fastCompileEntireModule(module, 0, err, 1024); + } // Called before a test function is run. def onTestRun(wf: WasmFunction, err: ErrorGen) { // TODO: move this to onFuncValidationFinish installStubForModule(wf.instance.module, X86_64Spc.setLazyCompileFor); @@ -407,8 +541,13 @@ class X86_64SpcLazyStrategy extends X86_64SpcStrategy { } // Two tiers: fast-int, frequently-executed functions are compiled with SPC. class X86_64DynamicStrategy extends X86_64SpcStrategy { + // Called after monitors have processed a module. + def onMonitorsFinish(module: Module, err: ErrorGen) { + if (FastIntTuning.useFastFunctions) fastCompileEntireModule(module, 0, err, 1024); + } // Called after a module is parsed. def onModuleFinish(module: Module, size: u32, err: ErrorGen) { + if (!hasMonitors && FastIntTuning.useFastFunctions) fastCompileEntireModule(module, size, err, 1024); installStubForModule(module, X86_64Spc.setTierUpFor); disableLazyNameDecodingDuringGC(module); if (Debug.runtime) { @@ -430,7 +569,7 @@ class X86_64DynamicStrategy extends X86_64SpcStrategy { } def onTierUp(wf: WasmFunction, pc: int) -> TargetOsrInfo { var module = wf.instance.module; - var compiler = newCompiler(module.filename); + var compiler = newCompiler(module.filename, false, null); if (!applyJitFilter(wf.instance.module, wf.decl, "osr")) { // OSR compile suppressed wf.decl.tierup_trigger = int.max; // no point in trying for a while @@ -459,10 +598,10 @@ class X86_64DynamicStrategy extends X86_64SpcStrategy { } } -def newCompiler(filename: string) -> X86_64SinglePassCompiler { +def newCompiler(filename: string, fast: bool, ic: X86_64InterpreterCode) -> X86_64SinglePassCompiler { var extensions = Extension.set.all; // TODO: all extensions enabled for compilation var limits = Limits.new(); - var compiler = X86_64SinglePassCompiler.new(extensions, limits, X86_64MasmRegs.CONFIG); + var compiler = X86_64SinglePassCompiler.new(ic, extensions, limits, X86_64MasmRegs.CONFIG, fast); return compiler; } def MINIMUM_CODE_SIZE = PAGE_SIZE_i; @@ -471,7 +610,7 @@ def allocateCodeForModule(module: Module, codeSize: int) { var codeSize = PAGE_SIZE_i * ((codeSize + PAGE_SIZE_i - 1) / PAGE_SIZE_i); // Allocate a read/write/execute mapping for code. var mapping = Mmap.reserve(u64.!(codeSize), Mmap.PROT_WRITE | Mmap.PROT_READ | Mmap.PROT_EXEC); - var code = X86_64SpcModuleCode.new(mapping); + var code = X86_64SpcModuleCode.new(mapping, false); module.target_module = TargetModule(code); RiRuntime.registerUserCode(code); code.keepAlive(); diff --git a/src/engine/x86-64/X86_64WhammProbeTrampoline.v3 b/src/engine/x86-64/X86_64WhammProbeTrampoline.v3 index 0446df2b9..b1f1a1d9f 100644 --- a/src/engine/x86-64/X86_64WhammProbeTrampoline.v3 +++ b/src/engine/x86-64/X86_64WhammProbeTrampoline.v3 @@ -120,7 +120,7 @@ def allocateCodeForTrampoline() { var code_size = PAGE_SIZE_i * FastIntTuning.whammProbeTrampolineNumPages; // Allocate a read/write/execute mapping for code. var mapping = Mmap.reserve(u64.!(code_size), Mmap.PROT_WRITE | Mmap.PROT_READ | Mmap.PROT_EXEC); - var code = X86_64SpcModuleCode.new(mapping); + var code = X86_64SpcModuleCode.new(mapping, false); RiRuntime.registerUserCode(code); code.keepAlive(); if (Trace.compiler) Trace.OUT.put2("Reserved 0x%x ... 0x%x for WhammProbe trampoline jit code", diff --git a/src/util/BytecodeVisitor.v3 b/src/util/BytecodeVisitor.v3 index 637b190af..70c05a701 100644 --- a/src/util/BytecodeVisitor.v3 +++ b/src/util/BytecodeVisitor.v3 @@ -70,6 +70,7 @@ class BytecodeVisitor { def visit_BR_TABLE (labels: Range) { visitControl(Opcode.BR_TABLE); } def visit_RETURN () { visitControl(Opcode.RETURN); } def visit_CALL (func_index: u31) { visitCallDirect(Opcode.CALL, func_index, false); } + def visit_FAST_CALL (fast_index: int, func_index: u31) { visit_CALL(func_index); } def visit_CALL_INDIRECT (sig_index: u31, table_index: u31) { visitCallIndirect(Opcode.CALL_INDIRECT, sig_index, table_index, false); } def visit_RETURN_CALL (func_index: u31) { visitCallDirect(Opcode.RETURN_CALL, func_index, true); } def visit_RETURN_CALL_INDIRECT(sig_index: u31, table_index: u31) { visitCallIndirect(Opcode.RETURN_CALL_INDIRECT, sig_index, table_index, true); } diff --git a/src/util/ErrorGen.v3 b/src/util/ErrorGen.v3 index 1782f0dc3..7868ef0a7 100644 --- a/src/util/ErrorGen.v3 +++ b/src/util/ErrorGen.v3 @@ -178,6 +178,18 @@ class ErrorGen(filename: string) { setc(WasmError.DUPLICATE_EXPORT, Strings.format1("duplicate export %d", export_index)); } + def DuplicateFastExport(func_index: int) { + setc(WasmError.DUPLICATE_FAST_EXPORT, + Strings.format1("function %d already exported under a different fast: name", func_index)); + } + def ImportedFastExport(func_index: int) { + setc(WasmError.IMPORTED_FAST_EXPORT, + Strings.format1("imported function %d cannot be exported under a fast: name", func_index)); + } + def IllegalFastFunctionOpcode(func_index: int, reason: string) { + setc(WasmError.ILLEGAL_FAST_FUNCTION_OPCODE, + Strings.format2("fast function %d uses %s", func_index, reason)); + } // XXX: factor out commonality of zero-byte checks def ExpectedMemoryIndexZeroByte(memory_index: byte) { setc(WasmError.EXPECTED_ZERO_BYTE, @@ -640,6 +652,9 @@ enum WasmError { OUT_OF_ORDER_SECTION, DUPLICATE_SECTION, DUPLICATE_EXPORT, + DUPLICATE_FAST_EXPORT, + IMPORTED_FAST_EXPORT, + ILLEGAL_FAST_FUNCTION_OPCODE, EXCEEDED_LIMIT, OOB_INDEX, PARSE_ERROR, diff --git a/test/fastcall/test21_inline_simple.wasm.flags b/test/fastcall/test21_inline_simple.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test21_inline_simple.wasm.flags +++ b/test/fastcall/test21_inline_simple.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test22_inline_control.wasm.flags b/test/fastcall/test22_inline_control.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test22_inline_control.wasm.flags +++ b/test/fastcall/test22_inline_control.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test23_inline_with_locals.wasm.flags b/test/fastcall/test23_inline_with_locals.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test23_inline_with_locals.wasm.flags +++ b/test/fastcall/test23_inline_with_locals.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test24_inline_depth2.wasm.flags b/test/fastcall/test24_inline_depth2.wasm.flags index d87b35653..3c975201d 100644 --- a/test/fastcall/test24_inline_depth2.wasm.flags +++ b/test/fastcall/test24_inline_depth2.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=2 +--fast-functions=true --fast-inline-max-depth=2 diff --git a/test/fastcall/test25_outcall_depth0.wasm.flags b/test/fastcall/test25_outcall_depth0.wasm.flags index 3920956e2..5af99f22a 100644 --- a/test/fastcall/test25_outcall_depth0.wasm.flags +++ b/test/fastcall/test25_outcall_depth0.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=0 +--fast-functions=true --fast-inline-max-depth=0 diff --git a/test/fastcall/test26_outcall_depth1.wasm.flags b/test/fastcall/test26_outcall_depth1.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test26_outcall_depth1.wasm.flags +++ b/test/fastcall/test26_outcall_depth1.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test27_outcall_stackframe.wasm.flags b/test/fastcall/test27_outcall_stackframe.wasm.flags index 92df8e667..893da2d53 100644 --- a/test/fastcall/test27_outcall_stackframe.wasm.flags +++ b/test/fastcall/test27_outcall_stackframe.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=0 \ No newline at end of file +--fast-functions=true --fast-inline-max-depth=0 \ No newline at end of file diff --git a/test/fastcall/test28_trap_mid_function.wasm.flags b/test/fastcall/test28_trap_mid_function.wasm.flags index 3920956e2..5af99f22a 100644 --- a/test/fastcall/test28_trap_mid_function.wasm.flags +++ b/test/fastcall/test28_trap_mid_function.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=0 +--fast-functions=true --fast-inline-max-depth=0 diff --git a/test/fastcall/test29_inline_then_outcall_trap.wasm.flags b/test/fastcall/test29_inline_then_outcall_trap.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test29_inline_then_outcall_trap.wasm.flags +++ b/test/fastcall/test29_inline_then_outcall_trap.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test30_inline_depth2_outcall_trap.wasm.flags b/test/fastcall/test30_inline_depth2_outcall_trap.wasm.flags index d87b35653..3c975201d 100644 --- a/test/fastcall/test30_inline_depth2_outcall_trap.wasm.flags +++ b/test/fastcall/test30_inline_depth2_outcall_trap.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=2 +--fast-functions=true --fast-inline-max-depth=2 diff --git a/test/fastcall/test31_fast_below_spc_trap.wasm.flags b/test/fastcall/test31_fast_below_spc_trap.wasm.flags index 3920956e2..5af99f22a 100644 --- a/test/fastcall/test31_fast_below_spc_trap.wasm.flags +++ b/test/fastcall/test31_fast_below_spc_trap.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=0 +--fast-functions=true --fast-inline-max-depth=0 diff --git a/test/fastcall/test32_inline_then_spc_trap.wasm.flags b/test/fastcall/test32_inline_then_spc_trap.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test32_inline_then_spc_trap.wasm.flags +++ b/test/fastcall/test32_inline_then_spc_trap.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test33_fast_self_trap.wasm.flags b/test/fastcall/test33_fast_self_trap.wasm.flags index 3920956e2..5af99f22a 100644 --- a/test/fastcall/test33_fast_self_trap.wasm.flags +++ b/test/fastcall/test33_fast_self_trap.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=0 +--fast-functions=true --fast-inline-max-depth=0 diff --git a/test/fastcall/test34_second_outcall_trap.wasm.flags b/test/fastcall/test34_second_outcall_trap.wasm.flags index 3920956e2..5af99f22a 100644 --- a/test/fastcall/test34_second_outcall_trap.wasm.flags +++ b/test/fastcall/test34_second_outcall_trap.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=0 +--fast-functions=true --fast-inline-max-depth=0 diff --git a/test/fastcall/test41_inline_loop.wasm.flags b/test/fastcall/test41_inline_loop.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test41_inline_loop.wasm.flags +++ b/test/fastcall/test41_inline_loop.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test42_inline_memory.wasm.flags b/test/fastcall/test42_inline_memory.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test42_inline_memory.wasm.flags +++ b/test/fastcall/test42_inline_memory.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test43_inline_global.wasm.flags b/test/fastcall/test43_inline_global.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test43_inline_global.wasm.flags +++ b/test/fastcall/test43_inline_global.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1 diff --git a/test/fastcall/test44_inline_depth3.wasm.flags b/test/fastcall/test44_inline_depth3.wasm.flags index a0d56bcb0..bb0ce3e84 100644 --- a/test/fastcall/test44_inline_depth3.wasm.flags +++ b/test/fastcall/test44_inline_depth3.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=3 +--fast-functions=true --fast-inline-max-depth=3 diff --git a/test/fastcall/test48_large_inline.wasm.flags b/test/fastcall/test48_large_inline.wasm.flags index 15e24ecd1..95334cb1b 100644 --- a/test/fastcall/test48_large_inline.wasm.flags +++ b/test/fastcall/test48_large_inline.wasm.flags @@ -1 +1 @@ ---fast-functions=true --inline-max-depth=1 +--fast-functions=true --fast-inline-max-depth=1