diff --git a/.gitignore b/.gitignore index d88dc51ca..2f823b2d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.profraw target wasm/target .idea diff --git a/Cargo.lock b/Cargo.lock index 510d7a487..d6b016fb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,13 +88,9 @@ checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "cc" -version = "1.2.55" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" -dependencies = [ - "find-msvc-tools", - "shlex", -] +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -401,12 +397,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - [[package]] name = "foldhash" version = "0.2.0" @@ -1177,12 +1167,6 @@ dependencies = [ "keccak", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "signature" version = "2.1.0" @@ -1513,18 +1497,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.39" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.39" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", diff --git a/resources/coverage/run_coverage.py b/resources/coverage/run_coverage.py index 34fc80ebb..bfd48fdca 100644 --- a/resources/coverage/run_coverage.py +++ b/resources/coverage/run_coverage.py @@ -1,3 +1,12 @@ +"""Coverage driver for CI and local runs. + +For ad-hoc reports on a subset of tests (e.g. module disk cache / funcache), you can run: + + LLVM_PROFILE_FILE='chialisp-%p-%m.profraw' RUSTFLAGS='-Cinstrument-coverage' \\ + cargo test --lib try_from_cache cache_key_stable funcache_without_dependency test_codegen_function_cache + +Then run grcov on the resulting ``*.profraw`` files (see the ``collect_coverage`` invocation below). +""" import os import json import shutil diff --git a/resources/tests/module/cache-test-1.clsp b/resources/tests/module/cache-test-1.clsp new file mode 100644 index 000000000..62dcba11b --- /dev/null +++ b/resources/tests/module/cache-test-1.clsp @@ -0,0 +1,31 @@ +(include *standard-cl-24*) + +(defconst C 33) +(defconst CC 99) + +(defun F (X) + (+ X C) + ) + +(defun G (X) + (sha256 X CC) + ) + +(defun H (X) + (sha256 1 X CC) + ) + +(defun FF (X Y) + (assign + Z (concat X Y) + (sha256 (F X) Y (fake-letbinding Z)) + ) + ) + +(defun-inline fake-letbinding (X) X) + +(defun GG (X Y) + (sha256 (G X) (H Y)) + ) + +(export (X Y) (c (FF X Y) (GG X Y))) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 60669e410..289eb6300 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -9,7 +9,7 @@ use num_bigint::ToBigInt; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; use crate::classic::clvm_tools::node_path::compose_paths; -use crate::compiler::clvm::{run, truthy}; +use crate::compiler::clvm::{run, sha256tree, truthy}; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ fold_m, join_vecs_to_string, list_to_cons, Binding, BindingPattern, BodyForm, CallSpec, @@ -23,10 +23,12 @@ use crate::compiler::frontend::{compile_bodyform, make_provides_set}; use crate::compiler::gensym::gensym; use crate::compiler::inline::{replace_in_inline, synthesize_args}; use crate::compiler::lambda::lambda_codegen; +use crate::compiler::optimize::depgraph::FunctionDependencyGraph; use crate::compiler::prims::{primapply, primcons, primquote}; use crate::compiler::runtypes::RunFailure; use crate::compiler::sexp::{decode_string, printable, SExp}; use crate::compiler::srcloc::Srcloc; +use crate::compiler::FunctionEntry; use crate::compiler::StartOfCodegenOptimization; use crate::compiler::{BasicCompileContext, CompileContextWrapper}; use crate::util::{toposort, u8_from_number, Number, TopoSortItem}; @@ -853,7 +855,7 @@ pub fn do_mod_codegen( &mut throwaway_symbols, optimizer, ); - let code = codegen(&mut context_wrapper.context, without_env, program)?; + let code = codegen(&mut context_wrapper.context, without_env, None, program)?; Ok(CompiledCode( program.loc.clone(), Rc::new(SExp::Cons( @@ -1054,9 +1056,111 @@ fn fail_if_present( } } +// Remove anything in the environment that isn't used by the caller and replace it with nil. +// the effect of this is to make only positional changes of things this function depends on +// matter. If a rearrangement occurs in things the caller doesn't depend on that leads to +// the same arrangement of things it does, then the output will be the same. +fn filter_env(used: &[Vec], nil: Rc, env: Rc) -> Rc { + match env.atomize() { + SExp::Atom(_l, n) => { + if used.iter().all(|u| *u != n) { + return nil.clone(); + } + env + } + SExp::Cons(l, a, b) => { + let new_a = filter_env(used, nil.clone(), a.clone()); + let new_b = filter_env(used, nil.clone(), b.clone()); + let old_a_ptr: *const SExp = &*a; + let old_b_ptr: *const SExp = &*b; + let new_a_ptr: *const SExp = &*new_a; + let new_b_ptr: *const SExp = &*new_b; + if old_a_ptr == new_a_ptr && old_b_ptr == new_b_ptr { + return env; + } + Rc::new(SExp::Cons(l.clone(), new_a, new_b)) + } + _ => env, + } +} + +// Make an sexp that contains the form of everything the caller depends on. Any change in these, +// such as whether a function is inline or not, the form of the environment or whether any constant +// is inline or tabled, will change this value and therefore its hash. If nohting relevant changed +// then this value will be the same and a previously generated version of the caller's function +// body can be used. +fn get_depended_on_forms( + compiler: &PrimaryCodegen, + loc: Srcloc, + mut env_shape: Rc, + used: &[Vec], +) -> Rc { + let append_with_name = |env_shape: Rc, name: &[u8], body: Rc| { + Rc::new(SExp::Cons( + loc.clone(), + Rc::new(SExp::Cons( + loc.clone(), + Rc::new(SExp::Atom(loc.clone(), name.to_vec())), + body, + )), + env_shape, + )) + }; + for name in used.iter() { + if let Some(c) = compiler.constants.get(name) { + env_shape = append_with_name( + env_shape, + name, + Rc::new(SExp::Cons( + c.loc(), + Rc::new(SExp::Nil(loc.clone())), + c.clone(), + )), + ); + } else if let Some(c) = compiler.tabled_constants.get(name) { + env_shape = append_with_name( + env_shape, + name, + Rc::new(SExp::Cons( + c.loc(), + Rc::new(SExp::Atom(loc.clone(), vec![1])), + c.clone(), + )), + ); + } else if let Some(i) = compiler.inlines.get(name) { + env_shape = append_with_name(env_shape, name, i.to_sexp()); + } + // Out of line functions are captured by the environment and it's ok for them + // to not have a full definition when generating code for other out of line functions. + } + env_shape +} + +fn get_function_cache_key( + compiler: &PrimaryCodegen, + dependency_graph: &FunctionDependencyGraph, + helper: &HelperForm, +) -> Vec { + let mut depends_on_set = HashSet::new(); + dependency_graph.get_full_depends_on(&mut depends_on_set, helper.name()); + let mut depends_on: Vec<_> = depends_on_set.into_iter().collect(); + depends_on.sort(); + // Collect depended on function forms + let filtered_env = filter_env( + &depends_on, + Rc::new(SExp::Nil(helper.loc())), + compiler.env.clone(), + ); + let depended_on = + get_depended_on_forms(compiler, helper.loc(), filtered_env.clone(), &depends_on); + let hashable = Rc::new(SExp::Cons(helper.loc(), helper.to_sexp(), depended_on)); + sha256tree(hashable) +} + fn codegen_( context: &mut BasicCompileContext, opts: Rc, + dependency_graph: Option<&FunctionDependencyGraph>, compiler: &PrimaryCodegen, h: &HelperForm, ) -> Result { @@ -1074,6 +1178,35 @@ fn codegen_( }, )) } else { + let check_already_present = |code| { + fail_if_present(defun.loc.clone(), &compiler.inlines, &defun.name, code) + .and_then(|code| { + fail_if_present(defun.loc.clone(), &compiler.defuns, &defun.name, code) + }) + }; + let cache_key = dependency_graph + .as_ref() + .filter(|_| context.funcache.is_some()) + .map(|d| get_function_cache_key(compiler, d, h)); + + if let Some(code) = cache_key.as_ref().and_then(|key| { + context + .funcache + .as_ref() + .and_then(|c| c.function_outputs.get(key).map(|e| e.code.clone())) + }) { + check_already_present(code.clone())?; + return Ok(compiler.add_defun( + &defun.name, + defun.orig_args.clone(), + DefunCall { + required_env: defun.args.clone(), + code, + }, + true, // Always take left env for now + )); + } + let updated_opts = opts .set_code_generator(compiler.clone()) .set_in_defun(true) @@ -1108,13 +1241,18 @@ fn codegen_( Rc::new(code.to_sexp()), ) }) - .and_then(|code| { - fail_if_present(defun.loc.clone(), &compiler.inlines, &defun.name, code) - }) - .and_then(|code| { - fail_if_present(defun.loc.clone(), &compiler.defuns, &defun.name, code) - }) + .and_then(check_already_present) .map(|code| { + if let (Some(fc), Some(hash)) = (&mut context.funcache, cache_key) { + fc.function_outputs.insert( + hash, + FunctionEntry { + code: code.clone(), + name: h.name().to_vec(), + }, + ); + } + compiler.add_defun( &defun.name, defun.orig_args.clone(), @@ -1859,6 +1997,7 @@ fn dummy_functions(compiler: &PrimaryCodegen) -> Result, + dependency_graph: Option<&FunctionDependencyGraph>, cmod: &CompileForm, ) -> Result { let mut start_of_codegen_optimization = StartOfCodegenOptimization { @@ -1904,7 +2043,7 @@ pub fn codegen( let to_process = code_generator.to_process.clone(); for f in to_process { - code_generator = codegen_(context, opts.clone(), &code_generator, &f)?; + code_generator = codegen_(context, opts.clone(), dependency_graph, &code_generator, &f)?; } // If stepping 23 or greater, we support no-env mode. diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index c87c15b47..c8aa62154 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -150,7 +150,7 @@ pub fn finish_compilation( let p3 = context.post_desugar_optimization(opts.clone(), p2)?; // generate code from AST, optionally with optimization - let generated = codegen(context, opts.clone(), &p3)?; + let generated = codegen(context, opts.clone(), None, &p3)?; let g2 = context.post_codegen_output_optimize(opts, generated)?; diff --git a/src/compiler/diskcache.rs b/src/compiler/diskcache.rs index e10f3c02b..ba189f450 100644 --- a/src/compiler/diskcache.rs +++ b/src/compiler/diskcache.rs @@ -60,3 +60,86 @@ pub fn set_cache_element( ) { set_cache_element_error(opts, cf, export_path, export_hex).ok(); } + +/// Exposes the cache-key segment used under `.chialisp//` (tests and tooling only). +#[cfg(test)] +pub fn module_cache_key_hex(cf: &CompileForm) -> String { + cache_key(cf) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::compiler::comptypes::{BodyForm, CompileForm, IncludeDesc, IncludeProcessType}; + use crate::compiler::sexp::SExp; + use crate::compiler::srcloc::Srcloc; + + fn empty_compileform(loc: Srcloc) -> CompileForm { + CompileForm { + loc: loc.clone(), + include_forms: Vec::new(), + args: Rc::new(SExp::Nil(loc.clone())), + helpers: Vec::new(), + exp: Rc::new(BodyForm::Quoted(SExp::Nil(loc.clone()))), + } + } + + #[test] + fn cache_key_stable_for_empty_includes() { + let loc = Srcloc::start(&"a.clsp".to_string()); + let cf = empty_compileform(loc); + let k = cache_key(&cf); + assert_eq!( + k, + "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a" + ); + } + + #[test] + fn cache_key_changes_with_concatenated_fingerprints() { + let loc = Srcloc::start(&"b.clsp".to_string()); + let mut cf = empty_compileform(loc.clone()); + let fp = |prefix: &[u8]| { + let mut a = [0u8; 32]; + a[..prefix.len()].copy_from_slice(prefix); + a + }; + let desc = |fp: [u8; 32]| IncludeDesc { + kw: loc.clone(), + nl: loc.clone(), + name: b"x".to_vec(), + kind: None, + fingerprint: fp, + }; + cf.include_forms.push(desc(fp(&[1, 2, 3]))); + let k1 = cache_key(&cf); + cf.include_forms.push(desc(fp(&[4, 5]))); + let k2 = cache_key(&cf); + assert_ne!(k1, k2); + cf.include_forms.truncate(1); + let k1_again = cache_key(&cf); + assert_eq!(k1, k1_again); + } + + #[test] + fn cache_key_main_fingerprint_style() { + let loc = Srcloc::start(&"c.clsp".to_string()); + let mut cf = empty_compileform(loc.clone()); + let mut main_fp = [0u8; 32]; + main_fp[0] = 0xab; + main_fp[1] = 0xcd; + cf.include_forms.push(IncludeDesc { + kw: loc.clone(), + nl: loc.clone(), + name: b"main".to_vec(), + kind: Some(IncludeProcessType::Compiled), + fingerprint: main_fp, + }); + let k = cache_key(&cf); + assert!(!k.is_empty()); + assert_ne!( + k, + "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a" + ); + } +} diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 8b37f62b1..82b484bbf 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -1497,7 +1497,12 @@ impl<'info> Evaluator { &mut symbols, optimizer, ); - let code = codegen(&mut context_wrapper.context, self.opts.clone(), program)?; + let code = codegen( + &mut context_wrapper.context, + self.opts.clone(), + None, + program, + )?; Ok(Rc::new(BodyForm::Quoted(code))) } BodyForm::Lambda(ldata) => self.enrich_lambda_site_info( diff --git a/src/compiler/mod.rs b/src/compiler/mod.rs index feb073684..ec598c48f 100644 --- a/src/compiler/mod.rs +++ b/src/compiler/mod.rs @@ -96,6 +96,17 @@ use crate::compiler::comptypes::{ use crate::compiler::optimize::Optimization; use crate::compiler::sexp::SExp; +#[derive(Clone)] +pub struct FunctionEntry { + pub name: Vec, + pub code: Rc, +} + +#[derive(Default)] +pub struct Funcache { + pub function_outputs: HashMap, FunctionEntry>, +} + /// An object which represents the standard set of mutable items passed down the /// stack when compiling chialisp. pub struct BasicCompileContext { @@ -103,6 +114,12 @@ pub struct BasicCompileContext { pub runner: Rc, pub symbols: HashMap, pub optimizer: Box, + /// Given the operative environment and a serialization of the helper, this is the generated + /// code from that helper. + /// + /// Since this is for speeding up optimization-time work, generation of the dependency graph + /// must follow desugaring. + pub funcache: Option, } impl BasicCompileContext { @@ -250,6 +267,7 @@ impl BasicCompileContext { runner, symbols, optimizer, + funcache: None, } } } diff --git a/src/compiler/optimize/deinline.rs b/src/compiler/optimize/deinline.rs index b071822c8..11ec87359 100644 --- a/src/compiler/optimize/deinline.rs +++ b/src/compiler/optimize/deinline.rs @@ -57,7 +57,7 @@ pub fn deinline_opt( let depgraph = FunctionDependencyGraph::new(&compileform); let mut best_compileform = compileform.clone(); - let generated_program = codegen(context, opts.clone(), &best_compileform)?; + let generated_program = codegen(context, opts.clone(), Some(&depgraph), &best_compileform)?; let mut metric = sexp_scale(&generated_program); let flip_helper = |h: &mut HelperForm| { if let HelperForm::Defun(inline, defun) = h { @@ -231,7 +231,8 @@ pub fn deinline_opt( continue; } - let maybe_smaller_program = codegen(context, opts.clone(), &compileform)?; + let maybe_smaller_program = + codegen(context, opts.clone(), Some(&depgraph), &compileform)?; let new_metric = sexp_scale(&maybe_smaller_program); // Don't keep this change if it made things worse. diff --git a/src/compiler/optimize/mod.rs b/src/compiler/optimize/mod.rs index a62620f9d..5d172bd43 100644 --- a/src/compiler/optimize/mod.rs +++ b/src/compiler/optimize/mod.rs @@ -334,7 +334,7 @@ fn constant_fun_result( let mut wrapper = CompileContextWrapper::new(allocator, runner.clone(), &mut symbols, optimizer); - if let Ok(code) = codegen(&mut wrapper.context, opts.clone(), &to_compile) { + if let Ok(code) = codegen(&mut wrapper.context, opts.clone(), None, &to_compile) { code } else { return None; diff --git a/src/tests/compiler/codegen_funcache.rs b/src/tests/compiler/codegen_funcache.rs new file mode 100644 index 000000000..bd7238472 --- /dev/null +++ b/src/tests/compiler/codegen_funcache.rs @@ -0,0 +1,443 @@ +use std::collections::HashMap; +use std::rc::Rc; + +use clvmr::Allocator; + +use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; + +use crate::compiler::clvm::run; +use crate::compiler::codegen::codegen; +use crate::compiler::compiler::{do_desugar, DefaultCompilerOpts}; +use crate::compiler::comptypes::{ + BodyForm, CompileErr, CompileForm, CompilerOpts, DefconstData, Export, FrontendOutput, + HelperForm, +}; +use crate::compiler::dialect::AcceptedDialect; +use crate::compiler::frontend::frontend; +use crate::compiler::optimize::above22::Strategy23; +use crate::compiler::optimize::depgraph::{DepgraphOptions, FunctionDependencyGraph}; +use crate::compiler::rename::rename_args_compileform; +use crate::compiler::sexp::{decode_string, parse_sexp, SExp}; +use crate::compiler::srcloc::Srcloc; +use crate::compiler::{BasicCompileContext, Funcache, FunctionEntry}; +use crate::tests::compiler::modules::TestModuleCompilerOpts; + +fn get_expected_outcome( + opts: Rc, + run_arg: Rc, +) -> Result, CompileErr> { + let mut allocator = Allocator::new(); + let runner = Rc::new(DefaultProgramRunner::new()); + let ff_program_base = parse_sexp( + Srcloc::start("*run-ff*"), + "(sha256 (+ 2 (q . 33)) 5 (concat 2 5))".bytes(), + ) + .unwrap()[0] + .clone(); + let gg_program_base = parse_sexp( + Srcloc::start("*run-gg*"), + "(sha256 (sha256 2 (q . 99)) (sha256 (q . 1) 5 (q . 99)))".bytes(), + ) + .unwrap()[0] + .clone(); + let ff_base_outcome = run( + &mut allocator, + runner.clone(), + opts.prim_map(), + ff_program_base, + run_arg.clone(), + None, + None, + )?; + let gg_base_outcome = run( + &mut allocator, + runner.clone(), + opts.prim_map(), + gg_program_base, + run_arg.clone(), + None, + None, + )?; + Ok(Rc::new(SExp::Cons( + run_arg.loc(), + ff_base_outcome.clone(), + gg_base_outcome.clone(), + ))) +} + +fn extract_helper_gen(map: &HashMap, FunctionEntry>, name: &str) -> Option> { + map.values() + .filter(|v| decode_string(&v.name) == name) + .next() + .map(|v| v.code.clone()) +} + +fn flip_inline(h: &HelperForm, toward: bool) -> Option { + if let HelperForm::Defun(_inl, defdata) = h { + return Some(HelperForm::Defun(toward, defdata.clone())); + } + None +} + +fn match_name_prefix(prefix: &str, h: &HelperForm) -> bool { + decode_string(h.name()).starts_with(prefix) +} + +fn untable_constant(h: &HelperForm) -> Option { + if let HelperForm::Defconstant(defc) = h { + return Some(HelperForm::Defconstant(DefconstData { + tabled: false, + ..defc.clone() + })); + } + None +} + +fn change_constant_value(h: &HelperForm, body: Rc) -> Option { + if let HelperForm::Defconstant(defc) = h { + return Some(HelperForm::Defconstant(DefconstData { + body, + ..defc.clone() + })); + } + None +} + +fn transform_helper Option>( + compileform: &mut CompileForm, + transform: F, +) { + for h in compileform.helpers.iter_mut() { + if let Some(res) = transform(&h) { + *h = res; + } + } +} + +fn diffmap(m1: &mut HashMap, m2: &HashMap) { + for key in m2.keys() { + m1.remove(key); + } +} + +#[test] +fn test_codegen_function_cache() { + let orig_opts: Rc = Rc::new(DefaultCompilerOpts::new(&"*test*".to_string())); + let orig_opts = orig_opts + .set_search_paths(&["resources/tests/module".to_string(), ".".to_string()]) + .set_stdenv(false) + .set_dialect(AcceptedDialect { + stepping: Some(25), + strict: true, + int_fix: true, + extra_numeric_constants: false, + }); + let fs_opts = TestModuleCompilerOpts::new(orig_opts); + let opts: Rc = Rc::new(fs_opts.clone()); + let (_, content) = opts + .read_new_file( + opts.filename(), + "resources/tests/module/cache-test-1.clsp".to_string(), + ) + .expect("ok"); + let parsed = parse_sexp(Srcloc::start(&opts.filename()), content.iter().cloned()).expect("ok"); + let program = frontend(opts.clone(), &parsed).expect("ok"); + let (mut compileform, exports) = if let FrontendOutput::Module(compileform, exports) = program { + (compileform, exports) + } else { + panic!(); + }; + + // Module style programs produce exports that are handled separately. They're turned into full + // programs in this way: + (compileform.args, compileform.exp) = if let Export::MainProgram(ep) = &exports[0] { + (ep.args.clone(), ep.expr.clone()) + } else { + panic!(); + }; + compileform = rename_args_compileform(&compileform).unwrap(); + + // + // Now we've got a compileform with some exports and some intermediate functions. + // check-1 depends on FF which depends on F, which depends on C + // check-2 depends on GG which depends on G and H, which depend on CC + // We should be able to change FF without changing the output of and G or H without + // chaning the output of GG unless the environment changes. + // We should be able to change C without changing the output of GG and CC without + // changing the output of FF. + // + // conversely, we should always pick up a change in C or F in FF. + // and CC, G, H in GG. + // + let runner = Rc::new(DefaultProgramRunner::new()); + let mut context = BasicCompileContext::new( + Allocator::new(), + runner.clone(), + HashMap::new(), + Box::new(Strategy23 {}), + ); + + let mut desugared = do_desugar(&compileform).unwrap(); + + // This enables caching. + context.funcache = Some(Funcache::default()); + + // Set the constant CC to not be tabled so we can test inlined constant changes. + transform_helper(&mut desugared, |h| { + if match_name_prefix("CC", h) { + return untable_constant(h); + } + None + }); + + let dependency_graph = FunctionDependencyGraph::new_with_options( + &desugared, + DepgraphOptions { + with_constants: true, + }, + ); + + let base_generated = Rc::new( + codegen( + &mut context, + opts.clone(), + Some(&dependency_graph), + &desugared, + ) + .unwrap(), + ); + let original_map = context.funcache.as_ref().unwrap().function_outputs.clone(); + let mut allocator = Allocator::new(); + let run_arg = parse_sexp(Srcloc::start("*args*"), b"(3 7)".iter().cloned()).unwrap()[0].clone(); + let expected_outcome = get_expected_outcome(opts.clone(), run_arg.clone()).unwrap(); + let run_outcome_base = run( + &mut allocator, + runner.clone(), + opts.prim_map(), + base_generated.clone(), + run_arg.clone(), + None, + None, + ) + .unwrap(); + assert_eq!(run_outcome_base, expected_outcome); + + // Ensure the cache is being used. We should have entries for FF, GG. + let want_set = vec!["FF", "GG", "F", "G", "H"]; + let names: Vec<_> = context + .funcache + .as_ref() + .map(|fc| { + fc.function_outputs + .values() + .map(|e| decode_string(&e.name)) + .collect() + }) + .unwrap_or_default(); + // maps names to their original hashes. + let original_name_map: HashMap<_, _> = context + .funcache + .as_ref() + .map(|fc| { + fc.function_outputs + .iter() + .map(|(k, v)| (decode_string(&v.name), k.to_vec())) + .collect() + }) + .unwrap_or_default(); + + // A let binding will have been desugared from FF. + // Show all the expected names are present in the cache along with the desugared assignment. + assert!(!names.is_empty()); + assert_eq!(names.len(), want_set.len() + 1); + assert_eq!(names.len(), original_name_map.len()); + + for name in want_set.iter() { + assert!(names.iter().any(|n| n == name)); + } + + let mut flipped_letbinding_program = desugared.clone(); + + transform_helper(&mut flipped_letbinding_program, |h| { + if match_name_prefix("letbinding_$_", h) { + return flip_inline(h, true); + } + None + }); + + // If the code wasn't regenerated in response to this, we'd have the same program. + let generated_flipped_let_binding = Rc::new( + codegen( + &mut context, + opts.clone(), + Some(&dependency_graph), + &flipped_letbinding_program, + ) + .unwrap(), + ); + let mut flipped_map = context.funcache.as_ref().unwrap().function_outputs.clone(); + diffmap(&mut flipped_map, &original_map); + + assert_ne!( + base_generated.to_string(), + generated_flipped_let_binding.to_string() + ); + + // Extract the generated FF and verify that it's different. + let old_ff = extract_helper_gen(&original_map, "FF").unwrap(); + let new_ff = extract_helper_gen(&flipped_map, "FF").unwrap(); + + let old_h = extract_helper_gen(&original_map, "H").unwrap(); + + // GG will have been different here because of the environment shift. + // We'll keep a representation of it to know for sure. + let old_gg = extract_helper_gen(&original_map, "GG").unwrap(); + let new_gg = extract_helper_gen(&flipped_map, "GG").unwrap(); + + // They aren't the same function code. + assert_ne!(old_ff, new_ff); + + // Verify the same run outcome. + let run_outcome_flipped = run( + &mut allocator, + runner.clone(), + opts.prim_map(), + generated_flipped_let_binding.clone(), + run_arg.clone(), + None, + None, + ) + .unwrap(); + assert_eq!(expected_outcome, run_outcome_flipped); + + // Swap dummy and the let binding. That should leave the environment in the same shape. + // If we've done things correctly, GG should be the same function in generated_flipped_let + // as in generated_double_flip. + let mut fake_flip_program = flipped_letbinding_program.clone(); + transform_helper(&mut fake_flip_program, |h| { + if match_name_prefix("fake-letbinding", h) { + return flip_inline(h, false); + } + None + }); + + let generated_double_flip = Rc::new( + codegen( + &mut context, + opts.clone(), + Some(&dependency_graph), + &fake_flip_program, + ) + .unwrap(), + ); + + // We'll zap the cache and do it again. The version we get for GG should be the same as in + // the original because we transformed the environment back to the same shape observed in the + // original while involving only functions that GG doesn't use. + context.funcache = Some(Funcache::default()); + let generated_double_flip_clean_cache = Rc::new( + codegen( + &mut context, + opts.clone(), + Some(&dependency_graph), + &fake_flip_program, + ) + .unwrap(), + ); + + // They're the same program, so we can use the clean cache below to check generated bodies. + assert_eq!(generated_double_flip, generated_double_flip_clean_cache); + + let fake_flip_map_clean_cache = context.funcache.as_ref().unwrap().function_outputs.clone(); + + let swap_ff = extract_helper_gen(&fake_flip_map_clean_cache, "FF").unwrap(); + let swap_gg = extract_helper_gen(&fake_flip_map_clean_cache, "GG").unwrap(); + + // FF should not be the same + assert_ne!(swap_ff, old_ff); + // GG should be the same, so we removed it by filtering. + assert_eq!(swap_gg, old_gg); + assert_ne!(swap_gg, new_gg); + + // Modify a constant affecting H. + let mut constant_diff_program = desugared.clone(); + transform_helper(&mut constant_diff_program, |h| { + if match_name_prefix("CC", h) { + return change_constant_value( + h, + Rc::new(BodyForm::Quoted(SExp::Atom(h.loc(), vec![37]))), + ); + } + None + }); + + let generated_diff_c = Rc::new( + codegen( + &mut context, + opts.clone(), + Some(&dependency_graph), + &constant_diff_program, + ) + .unwrap(), + ); + + // Clobber cache and generate again so we can see exactly what was generated. + context.funcache = Some(Funcache::default()); + let generated_diff_c_clean = Rc::new( + codegen( + &mut context, + opts.clone(), + Some(&dependency_graph), + &constant_diff_program, + ) + .unwrap(), + ); + let generated_diff_c_map_clean = context.funcache.as_ref().unwrap().function_outputs.clone(); + assert_eq!(generated_diff_c, generated_diff_c_clean); + + // // Now we have a different value of GG, but same value of FF. + let diffcc_ff = extract_helper_gen(&generated_diff_c_map_clean, "FF").unwrap(); + let diffcc_gg = extract_helper_gen(&generated_diff_c_map_clean, "GG").unwrap(); + let diffcc_h = extract_helper_gen(&generated_diff_c_map_clean, "H").unwrap(); + + assert_eq!(diffcc_ff, old_ff); + assert_eq!(diffcc_gg, old_gg); + assert_ne!(diffcc_h, old_h); +} + +/// `Funcache` is only consulted when a `FunctionDependencyGraph` is passed into `codegen`. +#[test] +fn funcache_without_dependency_graph_does_not_store_entries() { + let opts: Rc = Rc::new(DefaultCompilerOpts::new("*fc-no-graph*")) + .set_dialect(AcceptedDialect { + stepping: Some(25), + strict: true, + int_fix: true, + extra_numeric_constants: false, + }); + let loc = Srcloc::start("*fc-no-graph*"); + let parsed = + parse_sexp(loc.clone(), "(mod (X) (defun F (Y) (+ Y 1)) (F X))".bytes()).expect("parse"); + let FrontendOutput::CompileForm(cf) = frontend(opts.clone(), &parsed).expect("frontend") else { + panic!("expected CompileForm"); + }; + let desugared = do_desugar(&cf).expect("desugar"); + let runner = Rc::new(DefaultProgramRunner::new()); + let mut context = BasicCompileContext::new( + Allocator::new(), + runner, + HashMap::new(), + Box::new(Strategy23 {}), + ); + context.funcache = Some(Funcache::default()); + codegen(&mut context, opts, None, &desugared).expect("codegen"); + assert!( + context + .funcache + .as_ref() + .expect("funcache") + .function_outputs + .is_empty(), + "without a dependency graph, per-function cache keys are not computed" + ); +} diff --git a/src/tests/compiler/mod.rs b/src/tests/compiler/mod.rs index 5b24a3107..2bf88fde0 100644 --- a/src/tests/compiler/mod.rs +++ b/src/tests/compiler/mod.rs @@ -7,10 +7,12 @@ use crate::compiler::srcloc::{Srcloc, Until}; mod assign; mod cldb; mod clvm; +mod codegen_funcache; mod compiler; mod evaluate; mod fuzz; mod fuzz_assign; +mod module_cache; mod modules; mod optimizer; mod preprocessor; diff --git a/src/tests/compiler/module_cache.rs b/src/tests/compiler/module_cache.rs new file mode 100644 index 000000000..77f511057 --- /dev/null +++ b/src/tests/compiler/module_cache.rs @@ -0,0 +1,163 @@ +//! Tests for module export disk cache (`try_from_cache`) and cache key wiring. + +use std::rc::Rc; + +use clvmr::Allocator; + +use crate::classic::clvm::__type_compatibility__::Stream; +use crate::classic::clvm::sexp::sexp_as_bin; +use crate::compiler::clvm::convert_to_clvm_rs; +use crate::compiler::compiler::{try_from_cache, DefaultCompilerOpts}; +use crate::compiler::comptypes::{ + BodyForm, CompileForm, CompilerOpts, CompilerOutput, Export, ExportFunctionDesc, + ExportProgramDesc, NameAndLoc, +}; +use crate::compiler::diskcache::module_cache_key_hex; +use crate::compiler::sexp::{parse_sexp, SExp}; +use crate::compiler::srcloc::Srcloc; +use crate::tests::compiler::modules::TestModuleCompilerOpts; + +const TEST_CLSP: &str = "module_cache_try.clsp"; + +fn sample_program_hex() -> String { + let mut allocator = Allocator::new(); + let loc = Srcloc::start(&"*hex*".to_string()); + let sexp = parse_sexp(loc.clone(), "(q . 99)".bytes()).expect("parse")[0].clone(); + let converted = convert_to_clvm_rs(&mut allocator, sexp).expect("convert"); + let mut stream = Stream::new(None); + stream.write(sexp_as_bin(&mut allocator, converted)); + stream.get_value().hex() +} + +fn minimal_module_compileform() -> CompileForm { + let loc = Srcloc::start(&TEST_CLSP.to_string()); + CompileForm { + loc: loc.clone(), + include_forms: Vec::new(), + args: Rc::new(SExp::Nil(loc.clone())), + helpers: Vec::new(), + exp: Rc::new(BodyForm::Quoted(SExp::Nil(loc.clone()))), + } +} + +fn main_export() -> Export { + let loc = Srcloc::start(&TEST_CLSP.to_string()); + Export::MainProgram(ExportProgramDesc { + loc: loc.clone(), + kw_loc: None, + args: Rc::new(SExp::Nil(loc.clone())), + expr: Rc::new(BodyForm::Quoted(SExp::Nil(loc))), + }) +} + +fn function_export(name: &[u8], as_name: Option>) -> Export { + let loc = Srcloc::start(&TEST_CLSP.to_string()); + Export::Function(Box::new(ExportFunctionDesc { + loc: loc.clone(), + kw_loc: None, + name: NameAndLoc { + value: name.to_vec(), + loc: None, + }, + as_loc: None, + as_name: as_name.map(|value| NameAndLoc { value, loc: None }), + })) +} + +fn cache_dir_for(cf: &CompileForm) -> String { + format!(".chialisp/{}/", module_cache_key_hex(cf)) +} + +#[test] +fn try_from_cache_returns_none_when_cache_missing() { + let cf = minimal_module_compileform(); + let orig_opts: Rc = Rc::new(DefaultCompilerOpts::new(TEST_CLSP)); + let wrapped = TestModuleCompilerOpts::new(orig_opts); + let opts: Rc = Rc::new(wrapped); + let out = try_from_cache(opts, &cf, &[main_export()]).expect("try_from_cache"); + assert!(out.is_none()); +} + +#[test] +fn try_from_cache_returns_none_when_second_export_missing() { + let cf = minimal_module_compileform(); + let hex = sample_program_hex(); + let orig_opts: Rc = Rc::new(DefaultCompilerOpts::new(TEST_CLSP)); + let wrapped = TestModuleCompilerOpts::new(orig_opts); + let prefix = cache_dir_for(&cf); + wrapped.set_file_content( + format!("{}{}", prefix, TEST_CLSP.replace(".clsp", ".hex")), + hex.as_bytes().to_vec(), + ); + let opts: Rc = Rc::new(wrapped); + let exports = vec![main_export(), function_export(b"F", None)]; + let out = try_from_cache(opts, &cf, &exports).expect("try_from_cache"); + assert!(out.is_none()); +} + +#[test] +fn try_from_cache_returns_none_on_invalid_cached_hex() { + let cf = minimal_module_compileform(); + let orig_opts: Rc = Rc::new(DefaultCompilerOpts::new(TEST_CLSP)); + let wrapped = TestModuleCompilerOpts::new(orig_opts); + let prefix = cache_dir_for(&cf); + wrapped.set_file_content( + format!("{}{}", prefix, TEST_CLSP.replace(".clsp", ".hex")), + b"not_valid_hex_clvm_zzzz".to_vec(), + ); + let opts: Rc = Rc::new(wrapped); + let out = try_from_cache(opts, &cf, &[main_export()]).expect("try_from_cache"); + assert!(out.is_none()); +} + +#[test] +fn try_from_cache_hits_when_main_program_hex_present() { + let cf = minimal_module_compileform(); + let hex = sample_program_hex(); + let orig_opts: Rc = Rc::new(DefaultCompilerOpts::new(TEST_CLSP)); + let wrapped = TestModuleCompilerOpts::new(orig_opts); + let prefix = cache_dir_for(&cf); + let main_hex_path = format!("{}{}", prefix, TEST_CLSP.replace(".clsp", ".hex")); + wrapped.set_file_content(main_hex_path.clone(), hex.as_bytes().to_vec()); + let opts: Rc = Rc::new(wrapped.clone()); + + let out = try_from_cache(opts, &cf, &[main_export()]).expect("try_from_cache"); + let Some(CompilerOutput::Module(mo)) = out else { + panic!("expected module output"); + }; + assert_eq!(mo.components.len(), 1); + assert_eq!(mo.components[0].shortname, b"program".to_vec()); + assert_eq!( + mo.components[0].filename, + TEST_CLSP.replace(".clsp", ".hex") + ); + + let hash_path = format!("{}_hash.hex", TEST_CLSP.trim_end_matches(".clsp")); + assert!( + wrapped.get_written_file(&hash_path).is_some(), + "expected treehash sidecar {hash_path}, got {:?}", + wrapped.list_written_files() + ); +} + +#[test] +fn try_from_cache_uses_as_name_for_hex_path() { + let cf = minimal_module_compileform(); + let hex = sample_program_hex(); + let orig_opts: Rc = Rc::new(DefaultCompilerOpts::new(TEST_CLSP)); + let wrapped = TestModuleCompilerOpts::new(orig_opts); + let prefix = cache_dir_for(&cf); + let export = function_export(b"F", Some(b"RenamedF".to_vec())); + // Matches `create_hex_output_path`: `_.hex` with func dotted with `hex`. + let rel_hex = format!("{}_{}.hex", TEST_CLSP.trim_end_matches(".clsp"), "RenamedF"); + let cache_path = format!("{prefix}{rel_hex}"); + wrapped.set_file_content(cache_path, hex.as_bytes().to_vec()); + let opts: Rc = Rc::new(wrapped); + + let out = try_from_cache(opts, &cf, &[export]).expect("try_from_cache"); + let Some(CompilerOutput::Module(mo)) = out else { + panic!("expected module output"); + }; + assert_eq!(mo.components.len(), 1); + assert_eq!(mo.components[0].shortname, b"F".to_vec()); +} diff --git a/src/tests/compiler/resolve.rs b/src/tests/compiler/resolve.rs index b0effde18..8c8ca6e91 100644 --- a/src/tests/compiler/resolve.rs +++ b/src/tests/compiler/resolve.rs @@ -41,13 +41,13 @@ fn compile_program_get_result( let loc = Srcloc::start("*resolve-test*"); let resolved = do_program_module_resolution(opts.clone(), loc.clone(), test_program)?; let desugared = do_desugar(&resolved)?; - let mut context = BasicCompileContext { - allocator: Allocator::new(), - runner: Rc::new(DefaultProgramRunner::new()), - symbols: HashMap::new(), - optimizer: get_optimizer(&loc, opts.clone())?, - }; - let generated = codegen(&mut context, opts.clone(), &desugared)?; + let mut context = BasicCompileContext::new( + Allocator::new(), + Rc::new(DefaultProgramRunner::new()), + HashMap::new(), + get_optimizer(&loc, opts.clone())?, + ); + let generated = codegen(&mut context, opts.clone(), None, &desugared)?; run( &mut context.allocator, context.runner.clone(), diff --git a/wasm/Cargo.lock b/wasm/Cargo.lock index 4c05135fb..b7fc6cca1 100644 --- a/wasm/Cargo.lock +++ b/wasm/Cargo.lock @@ -116,13 +116,13 @@ dependencies = [ [[package]] name = "chia-bls" -version = "0.36.1" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f02cbfd038d9050d45edbe8f38e09391c73479c0cca5b37925daf48c4d4fcd4" +checksum = "8663dc7139234c60fccddc0babeb5b0a0a55b14943d74c73ac8aad130c5b79e3" dependencies = [ "blst", - "chia-sha2 0.36.1", - "chia-traits 0.36.1", + "chia-sha2 0.42.0", + "chia-traits 0.42.0", "hex", "hkdf", "linked-hash-map", @@ -141,9 +141,9 @@ dependencies = [ [[package]] name = "chia-sha2" -version = "0.36.1" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0934b0d6b878f29ba6c958e56e4b7158f9e687c200ffdca141dbc408a5cce42e" +checksum = "d6636ca8bba852fc516eacf01b2c3964b6b290359e7d1e89b950e6754e2a1082" dependencies = [ "sha2", ] @@ -161,12 +161,12 @@ dependencies = [ [[package]] name = "chia-traits" -version = "0.36.1" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4922b447b2d8418213948af1a448c3ca7b84e149b51b2c87a2e00e80bb19b0" +checksum = "3517dcf2b743672c78339ef8d222fd4493ae48741c35dd5ffb2d9dba90417115" dependencies = [ - "chia-sha2 0.36.1", - "chia_streamable_macro 0.36.1", + "chia-sha2 0.42.0", + "chia_streamable_macro 0.42.0", "thiserror", ] @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "chia_streamable_macro" -version = "0.36.1" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b60cefc5fe39f695816d42a327cbefad3d6d6a8ecadad1b58d7507067c25da8" +checksum = "41f85fb91e5ddba8003c4c4ad13c7914b1ff733d92136805c6566de227321d49" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -199,7 +199,7 @@ name = "chialisp" version = "0.4.3" dependencies = [ "binascii", - "chia-bls 0.36.1", + "chia-bls 0.42.0", "clvmr", "do-notation", "getrandom 0.2.15", @@ -1133,9 +1133,9 @@ checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "version_check"