diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index e791c092c2ab..04b27f2c2541 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -678,6 +678,70 @@ impl<'module_environment> FuncEnvironment<'module_environment> { self.fuel_check(builder); } + /// Consumes `units * cost_per_unit` fuel, saturating the charge at + /// `i64::MAX` so that an oversized unsigned operand cannot wrap around and + /// add fuel instead. + fn consume_variable_fuel( + &mut self, + builder: &mut FunctionBuilder<'_>, + units: ir::Value, + cost_per_unit: u8, + ) { + let may_exceed_i64_max = match builder.func.dfg.value_type(units) { + I32 => false, + I64 => true, + ty => unreachable!("unsupported variable fuel unit type: {ty}"), + }; + self.consume_variable_fuel_impl(builder, units, cost_per_unit, may_exceed_i64_max); + } + + /// Like [`Self::consume_variable_fuel`], but for a unit count whose product + /// with `cost_per_unit` is statically known to fit in an `i64`. + fn consume_bounded_variable_fuel( + &mut self, + builder: &mut FunctionBuilder<'_>, + units: ir::Value, + cost_per_unit: u8, + ) { + self.consume_variable_fuel_impl(builder, units, cost_per_unit, false); + } + + fn consume_variable_fuel_impl( + &mut self, + builder: &mut FunctionBuilder<'_>, + units: ir::Value, + cost_per_unit: u8, + may_exceed_i64_max: bool, + ) { + if !self.tunables.consume_fuel || cost_per_unit == 0 { + return; + } + + let units = match builder.func.dfg.value_type(units) { + I32 => builder.ins().uextend(I64, units), + I64 => units, + ty => unreachable!("unsupported variable fuel unit type: {ty}"), + }; + let fuel = if cost_per_unit == 1 { + units + } else { + builder.ins().imul_imm_s(units, i64::from(cost_per_unit)) + }; + let fuel = if may_exceed_i64_max { + let max = builder.ins().iconst(I64, i64::MAX); + let max_units = builder + .ins() + .iconst(I64, i64::MAX / i64::from(cost_per_unit)); + let saturate = builder + .ins() + .icmp(IntCC::UnsignedGreaterThan, units, max_units); + builder.ins().select(saturate, max, fuel) + } else { + fuel + }; + self.manual_fuel_check(builder, fuel); + } + fn epoch_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) { debug_assert!(self.epoch_deadline_var.is_reserved_value()); self.epoch_deadline_var = builder.declare_var(ir::types::I64); @@ -2610,6 +2674,11 @@ impl FuncEnvironment<'_> { self.table_vmctx_and_defined_index(&mut pos, table_index); let index_type = table.idx_type; let delta64 = self.cast_index_to_i64(&mut pos, delta, index_type); + let cost = self + .tunables + .operator_cost + .variable() + .table_grow_per_element; // Call out to the host to perform the actual growth of the underlying // table. This will initialize table slots as all null. Afterwards the @@ -2633,22 +2702,48 @@ impl FuncEnvironment<'_> { // Conditionally call that on growth success, and otherwise fall through // to continue to yield -1 for this growth operation. let current_block = builder.current_block().unwrap(); + let failed_block = builder.create_block(); let fill_block = builder.create_block(); let done_block = builder.create_block(); - builder.insert_block_after(fill_block, current_block); + builder.insert_block_after(failed_block, current_block); + builder.insert_block_after(fill_block, failed_block); builder.insert_block_after(done_block, fill_block); + // Commit the operator's flat cost before branching so translating the + // failed path cannot consume fuel that also belongs to the success path. + if self.tunables.consume_fuel { + self.fuel_increment_var(builder); + } let failure = builder.ins().iconst(index_type_to_ir_type(index_type), -1); let failed = builder.ins().icmp(IntCC::Equal, result_idx, failure); - builder.ins().brif(failed, done_block, &[], fill_block, &[]); + builder + .ins() + .brif(failed, failed_block, &[], fill_block, &[]); + + // A failed attempt performs no initialization loop, but still charge + // for the requested growth so repeated failures are not free. + builder.switch_to_block(failed_block); + self.consume_variable_fuel(builder, delta, cost); + builder.ins().jump(done_block, &[]); builder.switch_to_block(fill_block); - self.translate_table_fill(builder, table_index, result_idx, init_value, delta)?; + self.translate_entity_fill( + builder, + CheckedEntity::Table { + table: table_index, + initialized: true, + }, + result_idx, + init_value, + delta, + cost, + )?; builder.ins().jump(done_block, &[]); builder.switch_to_block(done_block); + builder.seal_block(failed_block); builder.seal_block(fill_block); builder.seal_block(done_block); @@ -2791,6 +2886,11 @@ impl FuncEnvironment<'_> { val: ir::Value, len: ir::Value, ) -> WasmResult<()> { + let cost = self + .tunables + .operator_cost + .variable() + .table_fill_per_element; self.translate_entity_fill( builder, CheckedEntity::Table { @@ -2800,6 +2900,7 @@ impl FuncEnvironment<'_> { dst, val, len, + cost, ) } @@ -2936,7 +3037,8 @@ impl FuncEnvironment<'_> { elem: ir::Value, len: ir::Value, ) -> WasmResult { - gc::translate_array_new(self, builder, array_type_index, elem, len) + let cost = self.tunables.operator_cost.variable().array_new_per_element; + gc::translate_array_new(self, builder, array_type_index, elem, len, cost) } pub fn translate_array_new_default( @@ -2945,7 +3047,12 @@ impl FuncEnvironment<'_> { array_type_index: TypeIndex, len: ir::Value, ) -> WasmResult { - gc::translate_array_new_default(self, builder, array_type_index, len) + let cost = self + .tunables + .operator_cost + .variable() + .array_new_default_per_element; + gc::translate_array_new_default(self, builder, array_type_index, len, cost) } pub fn translate_array_new_fixed( @@ -2967,6 +3074,11 @@ impl FuncEnvironment<'_> { ) -> WasmResult { let interned_type_index = self.module.types[array_type_index].unwrap_module_type_index(); let array_layout = self.array_layout(interned_type_index)?.clone(); + let cost = self + .tunables + .operator_cost + .variable() + .array_new_data_per_element; gc::translate_array_new_entity( self, builder, @@ -2977,6 +3089,7 @@ impl FuncEnvironment<'_> { }, data_offset, len, + cost, ) } @@ -2988,6 +3101,11 @@ impl FuncEnvironment<'_> { elem_offset: ir::Value, len: ir::Value, ) -> WasmResult { + let cost = self + .tunables + .operator_cost + .variable() + .array_new_elem_per_element; gc::translate_array_new_entity( self, builder, @@ -2995,6 +3113,7 @@ impl FuncEnvironment<'_> { CheckedEntity::Elem(elem_index), elem_offset, len, + cost, ) } @@ -3011,6 +3130,11 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let dst_ty = self.module.types[dst_array_type_index].unwrap_module_type_index(); let src_ty = self.module.types[src_array_type_index].unwrap_module_type_index(); + let cost = self + .tunables + .operator_cost + .variable() + .array_copy_per_element; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3026,6 +3150,7 @@ impl FuncEnvironment<'_> { dst_index, src_index, len, + cost, ) } @@ -3039,6 +3164,11 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let ty = self.module.types[array_type_index].unwrap_module_type_index(); + let cost = self + .tunables + .operator_cost + .variable() + .array_fill_per_element; self.translate_entity_fill( builder, CheckedEntity::Array { @@ -3049,6 +3179,7 @@ impl FuncEnvironment<'_> { index, value, len, + cost, ) } @@ -3064,6 +3195,11 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let ty = self.module.types[array_type_index].unwrap_module_type_index(); let array_layout = self.array_layout(ty)?.clone(); + let cost = self + .tunables + .operator_cost + .variable() + .array_init_data_per_element; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3078,6 +3214,7 @@ impl FuncEnvironment<'_> { dst_index, data_offset, len, + cost, ) } @@ -3092,6 +3229,11 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let ty = self.module.types[array_type_index].unwrap_module_type_index(); + let cost = self + .tunables + .operator_cost + .variable() + .array_init_elem_per_element; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3103,6 +3245,7 @@ impl FuncEnvironment<'_> { dst_index, elem_offset, len, + cost, ) } @@ -3474,6 +3617,10 @@ impl FuncEnvironment<'_> { self.memory_vmctx_and_defined_index(&mut pos, index); let index_type = self.memory(index).idx_type; + let cost = self.tunables.operator_cost.variable().memory_grow_per_page; + drop(pos); + self.consume_variable_fuel(builder, val, cost); + let mut pos = builder.cursor(); let val = self.cast_index_to_i64(&mut pos, val, index_type); let call_inst = pos .ins() @@ -3558,7 +3705,8 @@ impl FuncEnvironment<'_> { src: ir::Value, len: ir::Value, ) -> WasmResult<()> { - self.translate_entity_copy(builder, dst_index, src_index, dst, src, len) + let cost = self.tunables.operator_cost.variable().memory_copy_per_byte; + self.translate_entity_copy(builder, dst_index, src_index, dst, src, len, cost) } /// Perform a raw bulk-memory-like libcall. @@ -3583,12 +3731,15 @@ impl FuncEnvironment<'_> { const_len: Some(bytes), src_entity, dst_entity, + fuel, .. } = op { if bytes <= INLINE_COPY_MAX_BYTES { if self.tunables.consume_fuel { - self.fuel_consumed += bytes as i64; + let units = bytes / u64::from(fuel.bytes_per_unit); + self.fuel_consumed += + i64::try_from(units * u64::from(fuel.cost_per_unit)).unwrap(); } let src_region = self.bulk_copy_alias_region(builder.func, src_entity); let dst_region = self.bulk_copy_alias_region(builder.func, dst_entity); @@ -3608,36 +3759,35 @@ impl FuncEnvironment<'_> { let pointer_type = self.pointer_type(); // Performs a raw call to the actual libcall, as dictated by the - // provided `op`. This unconditionally inserts epoch/fuel checks for all - // calls. + // provided `op`. This inserts configured epoch/fuel checks before the + // call. let raw_call = - |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &_| { + |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &BulkOp| { if env.tunables.epoch_interruption { env.epoch_check(builder); } + let fuel = op.fuel(); + if env.tunables.consume_fuel && fuel.cost_per_unit != 0 { + let byte_len = op.len(); + debug_assert!(fuel.bytes_per_unit.is_power_of_two()); + let units = if fuel.bytes_per_unit == 1 { + byte_len + } else { + builder + .ins() + .ushr_imm_u(byte_len, i64::from(fuel.bytes_per_unit.trailing_zeros())) + }; + // With fuel enabled all calls emitted below are limited to + // `UNINTERRUPTABLE_CHUNK_SIZE`, so this multiplication + // cannot exceed `i64::MAX` even at the maximum `u8` rate. + env.consume_bounded_variable_fuel(builder, units, fuel.cost_per_unit); + } match *op { BulkOp::MemoryCopy { dst, src, len, .. } => { - if env.tunables.consume_fuel { - // Note that fuel is always a 64-bit counter. - let fuel_consumed = match env.pointer_type() { - ir::types::I32 => builder.ins().uextend(ir::types::I64, len), - ir::types::I64 => len, - _ => unreachable!(), - }; - env.manual_fuel_check(builder, fuel_consumed); - } let memory_copy = env.builtin_functions.memory_copy(&mut builder.func); builder.ins().call(memory_copy, &[vmctx, dst, src, len]); } - BulkOp::MemoryFill { dst, val, len } => { - if env.tunables.consume_fuel { - let fuel_consumed = match env.pointer_type() { - ir::types::I32 => builder.ins().uextend(ir::types::I64, len), - ir::types::I64 => len, - _ => unreachable!(), - }; - env.manual_fuel_check(builder, fuel_consumed); - } + BulkOp::MemoryFill { dst, val, len, .. } => { let memory_fill = env.builtin_functions.memory_fill(&mut builder.func); builder.ins().call(memory_fill, &[vmctx, dst, val, len]); } @@ -3851,6 +4001,7 @@ impl FuncEnvironment<'_> { dst: ir::Value, val: ir::Value, len: ir::Value, + cost_per_unit: u8, ) -> WasmResult<()> { let entity = entity.into(); let idx_type = entity.index_type(self); @@ -3871,11 +4022,22 @@ impl FuncEnvironment<'_> { dst: raw_dst_addr, val, len: len_ptr, + fuel: BulkFuel { + cost_per_unit, + bytes_per_unit: 1, + }, }, ); } CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => { - self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr)?; + self.emit_raw_array_or_table_fill( + builder, + entity, + raw_dst_addr, + val, + len_ptr, + cost_per_unit, + )?; } // Not allowed to be written to in wasm. CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => { @@ -3907,6 +4069,7 @@ impl FuncEnvironment<'_> { dst_elem_addr: ir::Value, value: ir::Value, copy_len: ir::Value, + cost_per_element: u8, ) -> WasmResult<()> { let pointer_ty = self.pointer_type(); @@ -3932,6 +4095,10 @@ impl FuncEnvironment<'_> { dst: dst_elem_addr, val: value, len: copy_byte_len, + fuel: BulkFuel { + cost_per_unit: cost_per_element, + bytes_per_unit: u8::try_from(elem_size).unwrap(), + }, }, ); return Ok(()); @@ -3998,9 +4165,9 @@ impl FuncEnvironment<'_> { // by the element size, then see if we turn again or exit. builder.switch_to_block(loop_block); let elem_addr = builder.append_block_param(loop_block, pointer_ty); - // Consume one unit of fuel per loop iteration. + // Consume the configured cost for this element before writing it. if self.tunables.consume_fuel { - self.fuel_consumed += 1; + self.fuel_consumed += i64::from(cost_per_element); } self.translate_loop_header(builder)?; match entity { @@ -4117,7 +4284,8 @@ impl FuncEnvironment<'_> { val: ir::Value, len: ir::Value, ) -> WasmResult<()> { - self.translate_entity_fill(builder, memory_index, dst, val, len) + let cost = self.tunables.operator_cost.variable().memory_fill_per_byte; + self.translate_entity_fill(builder, memory_index, dst, val, len, cost) } pub fn translate_memory_init( @@ -4130,6 +4298,7 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let seg_index = DataIndex::from_u32(seg_index); + let cost = self.tunables.operator_cost.variable().memory_init_per_byte; self.translate_entity_copy( builder, memory_index, @@ -4140,6 +4309,7 @@ impl FuncEnvironment<'_> { dst, src, len, + cost, ) } @@ -4190,6 +4360,7 @@ impl FuncEnvironment<'_> { dst: ir::Value, src: ir::Value, len: ir::Value, + cost_per_unit: u8, ) -> WasmResult<()> { let dst_entity = dst_entity.into(); let src_entity = src_entity.into(); @@ -4251,6 +4422,10 @@ impl FuncEnvironment<'_> { const_len: const_count, src_entity, dst_entity, + fuel: BulkFuel { + cost_per_unit, + bytes_per_unit: 1, + }, }, ); Ok(()) @@ -4268,6 +4443,7 @@ impl FuncEnvironment<'_> { len_ptr, src, const_count, + cost_per_unit, ), // Cannot copy into a data or element segment in wasm. @@ -4478,6 +4654,11 @@ impl FuncEnvironment<'_> { src: ir::Value, len: ir::Value, ) -> WasmResult<()> { + let cost = self + .tunables + .operator_cost + .variable() + .table_copy_per_element; self.translate_entity_copy( builder, CheckedEntity::Table { @@ -4491,6 +4672,7 @@ impl FuncEnvironment<'_> { dst, src, len, + cost, ) } @@ -4520,6 +4702,7 @@ impl FuncEnvironment<'_> { copy_len: ir::Value, src_index: ir::Value, const_count: Option, + cost_per_element: u8, ) -> WasmResult<()> { let pointer_type = self.pointer_type(); assert_eq!(builder.func.dfg.value_type(dst_elem_addr), pointer_type); @@ -4607,6 +4790,10 @@ impl FuncEnvironment<'_> { const_len, src_entity, dst_entity, + fuel: BulkFuel { + cost_per_unit: cost_per_element, + bytes_per_unit: u8::try_from(dst_element_size).unwrap(), + }, }, ); return Ok(()); @@ -4623,6 +4810,7 @@ impl FuncEnvironment<'_> { src_elem_addr, copy_len, src_index, + cost_per_element, &|this, builder, dst, src, src_index| { let write_ty = dst_entity.storage_type(this); let val = match src_entity { @@ -4829,6 +5017,7 @@ impl FuncEnvironment<'_> { src_elem_addr: ir::Value, copy_len: ir::Value, src_index: ir::Value, + cost_per_element: u8, copy_one: &dyn Fn( &mut Self, &mut FunctionBuilder<'_>, @@ -4976,9 +5165,9 @@ impl FuncEnvironment<'_> { let src_cur = builder.append_block_param(forward_block, self.pointer_type()); let src_index = builder.append_block_param(forward_block, src_index_ty); let forward_keepalives = append_keepalive_params(builder, forward_block); - // Consume a single unit of fuel on each iteration of the loop. + // Consume the configured cost for this element before copying it. if self.tunables.consume_fuel { - self.fuel_consumed += 1; + self.fuel_consumed += i64::from(cost_per_element); } self.translate_loop_header(builder)?; copy_one(self, builder, dst_cur, src_cur, src_index)?; @@ -5007,7 +5196,7 @@ impl FuncEnvironment<'_> { let src_index = builder.append_block_param(backwards_block, src_index_ty); let backward_keepalives = append_keepalive_params(builder, backwards_block); if self.tunables.consume_fuel { - self.fuel_consumed += 1; + self.fuel_consumed += i64::from(cost_per_element); } self.translate_loop_header(builder)?; let dst_cur = { @@ -5060,6 +5249,11 @@ impl FuncEnvironment<'_> { src: ir::Value, len: ir::Value, ) -> WasmResult<()> { + let cost = self + .tunables + .operator_cost + .variable() + .table_init_per_element; self.translate_entity_copy( builder, CheckedEntity::Table { @@ -5070,6 +5264,7 @@ impl FuncEnvironment<'_> { dst, src, len, + cost, ) } @@ -5968,6 +6163,11 @@ impl FuncEnvironment<'_> { index_type_to_ir_type(ty.idx_type), ty.limits.min.cast_signed(), ); + let cost = self + .tunables + .operator_cost + .variable() + .table_fill_per_element; self.translate_entity_fill( builder, CheckedEntity::Table { @@ -5977,6 +6177,7 @@ impl FuncEnvironment<'_> { dst, val, len, + cost, ) } @@ -6007,6 +6208,12 @@ impl FuncEnvironment<'_> { offset, segment_len, )?; + let cost = self + .tunables + .operator_cost + .variable() + .table_init_per_element; + self.consume_variable_fuel(builder, segment_len, cost); // Re-use the `table.set` translation for making this a simple function // to define. That re-executes the bounds check which is a bit @@ -6078,7 +6285,8 @@ impl FuncEnvironment<'_> { // Model the initialization here as a `memory.init`. let len = self.load_runtime_data_length(builder, data); let start = builder.ins().iconst(I32, 0); - self.translate_entity_copy(builder, memory, data, offset, start, len)?; + let cost = self.tunables.operator_cost.variable().memory_init_per_byte; + self.translate_entity_copy(builder, memory, data, offset, start, len, cost)?; // Finalize control-flow for the `MemorySegmentOffset::Static` case // above. @@ -6248,6 +6456,7 @@ enum BulkOp { const_len: Option, src_entity: CheckedEntity, dst_entity: CheckedEntity, + fuel: BulkFuel, }, /// A `memory.fill` operation, setting all bytes of `dst` to `val`. @@ -6260,9 +6469,31 @@ enum BulkOp { dst: ir::Value, val: ir::Value, len: ir::Value, + fuel: BulkFuel, }, } +impl BulkOp { + fn len(&self) -> ir::Value { + match self { + BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => *len, + } + } + + fn fuel(&self) -> BulkFuel { + match self { + BulkOp::MemoryCopy { fuel, .. } | BulkOp::MemoryFill { fuel, .. } => *fuel, + } + } +} + +#[derive(Copy, Clone)] +struct BulkFuel { + cost_per_unit: u8, + /// Number of copied bytes represented by one billable unit. + bytes_per_unit: u8, +} + /// A list of entities which can participate in various kinds of bulk operations /// in wasm. /// diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index e52e80a3581e..ce984ce7aec8 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -771,6 +771,7 @@ pub fn translate_array_new( array_type_index: TypeIndex, elem: ir::Value, len: ir::Value, + cost_per_element: u8, ) -> WasmResult { log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})"); let result = @@ -787,6 +788,7 @@ pub fn translate_array_new( zero, elem, len, + cost_per_element, )?; log::trace!("translate_array_new(..) -> {result:?}"); Ok(result) @@ -797,6 +799,7 @@ pub fn translate_array_new_default( builder: &mut FunctionBuilder, array_type_index: TypeIndex, len: ir::Value, + cost_per_element: u8, ) -> WasmResult { log::trace!("translate_array_new_default({array_type_index:?}, {len:?})"); @@ -817,6 +820,7 @@ pub fn translate_array_new_default( zero, elem, len, + cost_per_element, )?; Ok(result) } @@ -1720,6 +1724,7 @@ pub fn translate_array_new_entity( entity: CheckedEntity, entity_offset: ir::Value, len: ir::Value, + cost_per_element: u8, ) -> WasmResult { // Before actually allocating this array first do a bounds-check on the // passive entity itself. @@ -1739,6 +1744,7 @@ pub fn translate_array_new_entity( dst, entity_offset, len, + cost_per_element, )?; Ok(array) diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs index 37efb27a0cf5..70ea869d644d 100644 --- a/crates/environ/src/tunables.rs +++ b/crates/environ/src/tunables.rs @@ -540,6 +540,89 @@ impl OperatorCostStrategy { OperatorCostStrategy::Default => default_operator_cost(op), } } + + /// Get the costs of work whose size is only known at runtime. + pub fn variable(&self) -> &VariableOperatorCost { + match self { + OperatorCostStrategy::Table(cost) => &cost.variable, + OperatorCostStrategy::Default => &DEFAULT_VARIABLE_OPERATOR_COST, + } + } +} + +const DEFAULT_VARIABLE_OPERATOR_COST: VariableOperatorCost = VariableOperatorCost::new(); + +/// Fuel costs for operators whose work is proportional to a runtime operand. +/// +/// These costs are charged in addition to the corresponding flat cost in +/// [`OperatorCost`]. +#[derive(Clone, Hash, Serialize, Deserialize, Debug, PartialEq, Eq)] +pub struct VariableOperatorCost { + /// Cost per byte copied by `memory.copy`. + pub memory_copy_per_byte: u8, + /// Cost per byte written by `memory.fill`. + pub memory_fill_per_byte: u8, + /// Cost per byte copied by `memory.init`. + pub memory_init_per_byte: u8, + /// Cost per page requested by `memory.grow`. + pub memory_grow_per_page: u8, + + /// Cost per element copied by `table.copy`. + pub table_copy_per_element: u8, + /// Cost per element written by `table.fill`. + pub table_fill_per_element: u8, + /// Cost per element copied by `table.init`. + pub table_init_per_element: u8, + /// Cost per element requested by `table.grow`. + pub table_grow_per_element: u8, + + /// Cost per element copied by `array.copy`. + pub array_copy_per_element: u8, + /// Cost per element written by `array.fill`. + pub array_fill_per_element: u8, + /// Cost per element initialized by `array.new_data`. + pub array_new_data_per_element: u8, + /// Cost per element initialized by `array.init_data`. + pub array_init_data_per_element: u8, + /// Cost per element initialized by `array.new_elem`. + pub array_new_elem_per_element: u8, + /// Cost per element initialized by `array.init_elem`. + pub array_init_elem_per_element: u8, + /// Cost per element initialized by `array.new_default`. + pub array_new_default_per_element: u8, + /// Cost per element initialized by `array.new`. + pub array_new_per_element: u8, +} + +impl VariableOperatorCost { + /// Creates the default variable-cost table. + pub const fn new() -> Self { + Self { + memory_copy_per_byte: 1, + memory_fill_per_byte: 1, + memory_init_per_byte: 1, + // `memory.grow` did not previously have a dynamic fuel charge. + memory_grow_per_page: 0, + table_copy_per_element: 1, + table_fill_per_element: 1, + table_init_per_element: 1, + table_grow_per_element: 1, + array_copy_per_element: 1, + array_fill_per_element: 1, + array_new_data_per_element: 1, + array_init_data_per_element: 1, + array_new_elem_per_element: 1, + array_init_elem_per_element: 1, + array_new_default_per_element: 1, + array_new_per_element: 1, + } + } +} + +impl Default for VariableOperatorCost { + fn default() -> Self { + Self::new() + } } const fn default_operator_cost(op: &Operator) -> i64 { @@ -608,6 +691,8 @@ macro_rules! define_operator_cost { $( pub $op: u8, )* + /// Costs for work whose size is only known at runtime. + pub variable: VariableOperatorCost, } impl OperatorCost { @@ -629,6 +714,7 @@ macro_rules! define_operator_cost { $( $op: default_cost!($op), )* + variable: VariableOperatorCost::new(), } } } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index ebf93ccdecf3..521070e7538b 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -649,6 +649,10 @@ impl Config { /// Configures the fuel cost of each WebAssembly operator. /// + /// In addition to each operator's flat cost, [`OperatorCost::variable`] + /// configures per-byte, per-element, and per-page costs for operators whose + /// work depends on a runtime operand. + /// /// This is only relevant when [`Config::consume_fuel`] is enabled. pub fn operator_cost(&mut self, cost: OperatorCost) -> &mut Self { self.tunables.operator_cost = Some(OperatorCostStrategy::table(cost)); diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index 175fd3703234..f3f0a3fb90d4 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -509,11 +509,11 @@ mod sync_nostd; #[cfg(not(feature = "std"))] use sync_nostd as sync; -pub use wasmtime_environ::OperatorCost; pub use wasmtime_environ::ToWasmtimeResult; #[doc(inline)] pub use wasmtime_environ::error; pub use wasmtime_environ::{FuncIndex, StaticModuleIndex}; +pub use wasmtime_environ::{OperatorCost, VariableOperatorCost}; // Only for use in `bindgen!`-generated code. #[doc(hidden)] diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs index 8a9c0d41ef4b..8d9073940a7e 100644 --- a/tests/all/fuel.rs +++ b/tests/all/fuel.rs @@ -693,3 +693,278 @@ fn custom_operator_cost(config: &mut Config) -> Result<()> { Ok(()) } + +#[wasmtime_test(wasm_features(bulk_memory, reference_types, gc, function_references))] +#[cfg_attr(miri, ignore)] +fn custom_variable_operator_cost(config: &mut Config) -> Result<()> { + config.consume_fuel(true); + + let mut op_cost = OperatorCost { + I32Const: 0, + I64Const: 0, + RefNull: 0, + LocalGet: 0, + LocalSet: 0, + MemoryCopy: 59, + MemoryFill: 0, + MemoryInit: 0, + MemoryGrow: 61, + TableCopy: 0, + TableFill: 67, + TableInit: 0, + TableGrow: 0, + ArrayCopy: 73, + ArrayFill: 0, + ArrayNewData: 71, + ArrayInitData: 0, + ArrayNewElem: 0, + ArrayInitElem: 0, + ArrayNewDefault: 0, + ArrayNew: 0, + ..Default::default() + }; + op_cost.variable = VariableOperatorCost { + memory_copy_per_byte: 2, + memory_fill_per_byte: 3, + memory_init_per_byte: 5, + memory_grow_per_page: 7, + table_copy_per_element: 11, + table_fill_per_element: 13, + table_init_per_element: 17, + table_grow_per_element: 19, + array_copy_per_element: 23, + array_fill_per_element: 29, + array_new_data_per_element: 31, + array_init_data_per_element: 37, + array_new_elem_per_element: 41, + array_init_elem_per_element: 43, + array_new_default_per_element: 47, + array_new_per_element: 53, + }; + config.operator_cost(op_cost.clone()); + + let wasm = wat::parse_str( + r#"(module + (type $i64_array (array (mut i64))) + (type $i32_array (array (mut i32))) + (type $ref_array (array (mut funcref))) + (memory 1 6) + (table 5 10 funcref) + (data $data "abcdefghijklmnopqrstuvwxyz") + (func $f) + (elem $elem func $f $f $f $f $f) + + (func (export "main") + (local $i64_values (ref null $i64_array)) + (local $i32_values (ref null $i32_array)) + (local $ref_values (ref null $ref_array)) + + i32.const 0 i32.const 0 i32.const 5 memory.copy + i32.const 0 i32.const 0 i32.const 5 memory.fill + i32.const 0 i32.const 0 i32.const 5 memory.init $data + i32.const 5 memory.grow drop + + i32.const 0 i32.const 0 i32.const 5 table.copy + i32.const 0 ref.null func i32.const 5 table.fill + i32.const 0 i32.const 0 i32.const 5 table.init $elem + ref.null func i32.const 5 table.grow drop + + i64.const 0 i32.const 5 array.new $i64_array + local.set $i64_values + i32.const 5 array.new_default $i64_array drop + i32.const 0 i32.const 5 array.new_data $i32_array $data + local.set $i32_values + i32.const 0 i32.const 5 array.new_elem $ref_array $elem + local.set $ref_values + + local.get $i64_values i32.const 0 + local.get $i64_values i32.const 0 + i32.const 5 array.copy $i64_array $i64_array + local.get $i64_values i32.const 0 i64.const 0 i32.const 5 + array.fill $i64_array + local.get $i32_values i32.const 0 i32.const 0 i32.const 5 + array.init_data $i32_array $data + local.get $ref_values i32.const 0 i32.const 0 i32.const 5 + array.init_elem $ref_array $elem) + )"#, + )?; + let engine = Engine::new(config)?; + let module = Module::new(&engine, wasm)?; + let mut store = Store::new(&engine, ()); + store.set_fuel(10_000)?; + let instance = Instance::new(&mut store, &module, &[])?; + let main = instance.get_typed_func::<(), ()>(&mut store, "main")?; + let initial_fuel = store.get_fuel()?; + main.call(&mut store, ())?; + let consumed = initial_fuel - store.get_fuel()?; + + const UNITS: u64 = 5; + let variable = &op_cost.variable; + let base_cost = u64::from(op_cost.MemoryCopy) + + u64::from(op_cost.MemoryGrow) + + u64::from(op_cost.TableFill) + + u64::from(op_cost.ArrayCopy) + + u64::from(op_cost.ArrayNewData); + let cost_of_execution = base_cost + + UNITS + * (u64::from(variable.memory_copy_per_byte) + + u64::from(variable.memory_fill_per_byte) + + u64::from(variable.memory_init_per_byte) + + u64::from(variable.memory_grow_per_page) + + u64::from(variable.table_copy_per_element) + + u64::from(variable.table_fill_per_element) + + u64::from(variable.table_init_per_element) + + u64::from(variable.table_grow_per_element) + + u64::from(variable.array_copy_per_element) + + u64::from(variable.array_fill_per_element) + + u64::from(variable.array_new_data_per_element) + + u64::from(variable.array_init_data_per_element) + + u64::from(variable.array_new_elem_per_element) + + u64::from(variable.array_init_elem_per_element) + + u64::from(variable.array_new_default_per_element) + + u64::from(variable.array_new_per_element)) + + 1; + assert_eq!(consumed, cost_of_execution); + + Ok(()) +} + +#[wasmtime_test(wasm_features(reference_types), strategies(not(Winch)))] +#[cfg_attr(miri, ignore)] +fn variable_operator_cost_failed_growth(config: &mut Config) -> Result<()> { + config.consume_fuel(true); + let mut op_cost = OperatorCost { + I32Const: 0, + RefNull: 0, + MemoryGrow: 0, + TableGrow: 0, + ..Default::default() + }; + op_cost.variable.memory_grow_per_page = 7; + op_cost.variable.table_grow_per_element = 19; + config.operator_cost(op_cost.clone()); + + let engine = Engine::new(config)?; + let module = Module::new( + &engine, + r#"(module + (memory 1 1) + (table 0 0 funcref) + (func (export "main") + ;; these exceed the max limits so they fail + i32.const 5 memory.grow drop + ref.null func i32.const 5 table.grow drop) + )"#, + )?; + let mut store = Store::new(&engine, ()); + store.set_fuel(1_000)?; + let instance = Instance::new(&mut store, &module, &[])?; + let main = instance.get_typed_func::<(), ()>(&mut store, "main")?; + + let initial_fuel = store.get_fuel()?; + main.call(&mut store, ())?; + let cost_of_execution = 5 * u64::from(op_cost.variable.memory_grow_per_page) + + 5 * u64::from(op_cost.variable.table_grow_per_element) + + 1; + assert_eq!(store.get_fuel()?, initial_fuel - cost_of_execution); + + Ok(()) +} + +#[wasmtime_test(wasm_features(bulk_memory), strategies(not(Winch)))] +#[cfg_attr(miri, ignore)] +fn variable_operator_cost_follows_bounds_check(config: &mut Config) -> Result<()> { + config.consume_fuel(true); + let op_cost = OperatorCost { + I32Const: 0, + MemoryFill: 0, + variable: VariableOperatorCost { + memory_fill_per_byte: 7, + ..Default::default() + }, + ..Default::default() + }; + config.operator_cost(op_cost); + + let engine = Engine::new(config)?; + let module = Module::new( + &engine, + r#"(module + (memory 1) + (func (export "main") + ;; out of bounds fill + i32.const 65535 i32.const 0 i32.const 5 memory.fill) + )"#, + )?; + let mut store = Store::new(&engine, ()); + store.set_fuel(1_000)?; + let instance = Instance::new(&mut store, &module, &[])?; + let main = instance.get_typed_func::<(), ()>(&mut store, "main")?; + + let initial_fuel = store.get_fuel()?; + let error = main.call(&mut store, ()).unwrap_err(); + assert_eq!(error.downcast::().unwrap(), Trap::MemoryOutOfBounds); + // The operation traps during bounds validation, before its five-byte + // variable charge or the pending function-entry unit is flushed. + assert_eq!(store.get_fuel()?, initial_fuel); + + Ok(()) +} + +#[wasmtime_test(wasm_features(memory64), strategies(not(Winch)))] +#[cfg_attr(miri, ignore)] +fn memory64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> { + config.consume_fuel(true); + let mut operator_cost = OperatorCost::default(); + operator_cost.variable.memory_grow_per_page = 2; + config.operator_cost(operator_cost); + + let engine = Engine::new(config)?; + let module = Module::new( + &engine, + r#"(module + (memory i64 0 0) + (func (export "grow") (param i64) (result i64) + local.get 0 memory.grow) + )"#, + )?; + let mut store = Store::new(&engine, ()); + store.set_fuel(10_000)?; + let instance = Instance::new(&mut store, &module, &[])?; + let grow = instance.get_typed_func::(&mut store, "grow")?; + + // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2. + let error = grow.call(&mut store, i64::MAX).unwrap_err(); + assert_eq!(error.downcast::().unwrap(), Trap::OutOfFuel); + + Ok(()) +} + +#[wasmtime_test(wasm_features(memory64, reference_types), strategies(not(Winch)))] +#[cfg_attr(miri, ignore)] +fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> { + config.consume_fuel(true); + let mut operator_cost = OperatorCost::default(); + operator_cost.variable.table_grow_per_element = 2; + config.operator_cost(operator_cost); + + let engine = Engine::new(config)?; + let module = Module::new( + &engine, + r#"(module + (table i64 0 0 funcref) + (func (export "grow") (param i64) (result i64) + ref.null func local.get 0 table.grow) + )"#, + )?; + let mut store = Store::new(&engine, ()); + store.set_fuel(10_000)?; + let instance = Instance::new(&mut store, &module, &[])?; + let grow = instance.get_typed_func::(&mut store, "grow")?; + + // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2. + let error = grow.call(&mut store, i64::MAX).unwrap_err(); + assert_eq!(error.downcast::().unwrap(), Trap::OutOfFuel); + + Ok(()) +}