//! Low-level compilation of an fused adapter function. //! //! This module is tasked with the top-level `compile` function which creates a //! single WebAssembly function which will perform the steps of the fused //! adapter for an `AdapterData` provided. This is the "meat" of compilation //! where the validation of the canonical ABI or similar all happens to //! translate arguments from one module to another. //! //! ## Traps and their ordering //! //! Currently this compiler is pretty "loose" about the ordering of precisely //! what trap happens where. The main reason for this is that to core wasm all //! traps are the same and for fused adapters if a trap happens no intermediate //! side effects are visible (as designed by the canonical ABI itself). For this //! it's important to note that some of the precise choices of control flow here //! can be somewhat arbitrary, an intentional decision. use crate::component::{ CanonicalAbiInfo, ComponentTypesBuilder, FixedEncoding as FE, FlatType, InterfaceType, StringEncoding, Transcode, TypeEnumIndex, TypeFlagsIndex, TypeListIndex, TypeOptionIndex, TypeRecordIndex, TypeResourceTableIndex, TypeResultIndex, TypeTupleIndex, TypeVariantIndex, VariantInfo, FLAG_MAY_ENTER, FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, }; use crate::fact::signature::Signature; use crate::fact::transcode::Transcoder; use crate::fact::traps::Trap; use crate::fact::{ AdapterData, Body, Context, Function, FunctionId, Helper, HelperLocation, HelperType, Module, Options, }; use crate::prelude::*; use crate::{FuncIndex, GlobalIndex}; use std::collections::HashMap; use std::mem; use std::ops::Range; use wasm_encoder::{BlockType, Encode, Instruction, Instruction::*, MemArg, ValType}; use wasmtime_component_util::{DiscriminantSize, FlagsSize}; const MAX_STRING_BYTE_LENGTH: u32 = 1 << 31; const UTF16_TAG: u32 = 1 << 31; /// This value is arbitrarily chosen and should be fine to change at any time, /// it just seemed like a halfway reasonable starting point. const INITIAL_FUEL: usize = 1_000; struct Compiler<'a, 'b> { types: &'a ComponentTypesBuilder, module: &'b mut Module<'a>, result: FunctionId, /// The encoded WebAssembly function body so far, not including locals. code: Vec<u8>, /// Total number of locals generated so far. nlocals: u32, /// Locals partitioned by type which are not currently in use. free_locals: HashMap<ValType, Vec<u32>>, /// Metadata about all `unreachable` trap instructions in this function and /// what the trap represents. The offset within `self.code` is recorded as /// well. traps: Vec<(usize, Trap)>, /// A heuristic which is intended to limit the size of a generated function /// to a certain maximum to avoid generating arbitrarily large functions. /// /// This fuel counter is decremented each time `translate` is called and /// when fuel is entirely consumed further translations, if necessary, will /// be done through calls to other functions in the module. This is intended /// to be a heuristic to split up the main function into theoretically /// reusable portions. fuel: usize, /// Indicates whether an "enter call" should be emitted in the generated /// function with a call to `Resource{Enter,Exit}Call` at the beginning and /// end of the function for tracking of information related to borrowed /// resources. emit_resource_call: bool, } pub(super) fn compile(module: &mut Module<'_>, adapter: &AdapterData) { let lower_sig = module.types.signature(&adapter.lower, Context::Lower); let lift_sig = module.types.signature(&adapter.lift, Context::Lift); let ty = module .core_types .function(&lower_sig.params, &lower_sig.results); let result = module .funcs .push(Function::new(Some(adapter.name.clone()), ty)); // If this type signature contains any borrowed resources then invocations // of enter/exit call for resource-related metadata tracking must be used. // It shouldn't matter whether the lower/lift signature is used here as both // should return the same answer. let emit_resource_call = module.types.contains_borrow_resource(&adapter.lower); assert_eq!( emit_resource_call, module.types.contains_borrow_resource(&adapter.lift) ); Compiler { types: module.types, module, code: Vec::new(), nlocals: lower_sig.params.len() as u32, free_locals: HashMap::new(), traps: Vec::new(), result, fuel: INITIAL_FUEL, emit_resource_call, } .compile_adapter(adapter, &lower_sig, &lift_sig) } /// Compiles a helper function as specified by the `Helper` configuration. /// /// This function is invoked when the translation process runs out of fuel for /// some prior function which enqueues a helper to get translated later. This /// translation function will perform one type translation as specified by /// `Helper` which can either be in the stack or memory for each side. pub(super) fn compile_helper(module: &mut Module<'_>, result: FunctionId, helper: Helper) { let mut nlocals = 0; let src_flat; let src = match helper.src.loc { // If the source is on the stack then it's specified in the parameters // to the function, so this creates the flattened representation and // then lists those as the locals with appropriate types for the source // values. HelperLocation::Stack => { src_flat = module .types .flatten_types(&helper.src.opts, usize::MAX, [helper.src.ty]) .unwrap() .iter() .enumerate() .map(|(i, ty)| (i as u32, *ty)) .collect::<Vec<_>>(); nlocals += src_flat.len() as u32; Source::Stack(Stack { locals: &src_flat, opts: &helper.src.opts, }) } // If the source is in memory then that's just propagated here as the // first local is the pointer to the source. HelperLocation::Memory => { nlocals += 1; Source::Memory(Memory { opts: &helper.src.opts, addr: TempLocal::new(0, helper.src.opts.ptr()), offset: 0, }) } }; let dst_flat; let dst = match helper.dst.loc { // This is the same as the stack-based source although `Destination` is // configured slightly differently. HelperLocation::Stack => { dst_flat = module .types .flatten_types(&helper.dst.opts, usize::MAX, [helper.dst.ty]) .unwrap(); Destination::Stack(&dst_flat, &helper.dst.opts) } // This is the same as a memory-based source but note that the address // of the destination is passed as the final parameter to the function. HelperLocation::Memory => { nlocals += 1; Destination::Memory(Memory { opts: &helper.dst.opts, addr: TempLocal::new(nlocals - 1, helper.dst.opts.ptr()), offset: 0, }) } }; let mut compiler = Compiler { types: module.types, module, code: Vec::new(), nlocals, free_locals: HashMap::new(), traps: Vec::new(), result, fuel: INITIAL_FUEL, // This is a helper function and only the top-level function is // responsible for emitting these intrinsic calls. emit_resource_call: false, }; compiler.translate(&helper.src.ty, &src, &helper.dst.ty, &dst); compiler.finish(); } /// Possible ways that a interface value is represented in the core wasm /// canonical ABI. enum Source<'a> { /// This value is stored on the "stack" in wasm locals. /// /// This could mean that it's inline from the parameters to the function or /// that after a function call the results were stored in locals and the /// locals are the inline results. Stack(Stack<'a>), /// This value is stored in linear memory described by the `Memory` /// structure. Memory(Memory<'a>), } /// Same as `Source` but for where values are translated into. enum Destination<'a> { /// This value is destined for the WebAssembly stack which means that /// results are simply pushed as we go along. /// /// The types listed are the types that are expected to be on the stack at /// the end of translation. Stack(&'a [ValType], &'a Options), /// This value is to be placed in linear memory described by `Memory`. Memory(Memory<'a>), } struct Stack<'a> { /// The locals that comprise a particular value. /// /// The length of this list represents the flattened list of types that make /// up the component value. Each list has the index of the local being /// accessed as well as the type of the local itself. locals: &'a [(u32, ValType)], /// The lifting/lowering options for where this stack of values comes from opts: &'a Options, } /// Representation of where a value is going to be stored in linear memory. struct Memory<'a> { /// The lifting/lowering options with memory configuration opts: &'a Options, /// The index of the local that contains the base address of where the /// storage is happening. addr: TempLocal, /// A "static" offset that will be baked into wasm instructions for where /// memory loads/stores happen. offset: u32, } impl Compiler<'_, '_> { fn compile_adapter( mut self, adapter: &AdapterData, lower_sig: &Signature, lift_sig: &Signature, ) { // Check the instance flags required for this trampoline. // // This inserts the initial check required by `canon_lower` that the // caller instance can be left and additionally checks the // flags on the callee if necessary whether it can be entered. self.trap_if_not_flag(adapter.lower.flags, FLAG_MAY_LEAVE, Trap::CannotLeave); if adapter.called_as_export { self.trap_if_not_flag(adapter.lift.flags, FLAG_MAY_ENTER, Trap::CannotEnter); self.set_flag(adapter.lift.flags, FLAG_MAY_ENTER, false); } else if self.module.debug { self.assert_not_flag( adapter.lift.flags, FLAG_MAY_ENTER, "may_enter should be unset", ); } if self.emit_resource_call { let enter = self.module.import_resource_enter_call(); self.instruction(Call(enter.as_u32())); } // Perform the translation of arguments. Note that `FLAG_MAY_LEAVE` is // cleared around this invocation for the callee as per the // `canon_lift` definition in the spec. Additionally note that the // precise ordering of traps here is not required since internal state // is not visible to either instance and a trap will "lock down" both // instances to no longer be visible. This means that we're free to // reorder lifts/lowers and flags and such as is necessary and // convenient here. // // TODO: if translation doesn't actually call any functions in either // instance then there's no need to set/clear the flag here and that can // be optimized away. self.set_flag(adapter.lift.flags, FLAG_MAY_LEAVE, false); let param_locals = lower_sig .params .iter() .enumerate() .map(|(i, ty)| (i as u32, *ty)) .collect::<Vec<_>>(); self.translate_params(adapter, ¶m_locals); self.set_flag(adapter.lift.flags, FLAG_MAY_LEAVE, true); // With all the arguments on the stack the actual target function is // now invoked. The core wasm results of the function are then placed // into locals for result translation afterwards. self.instruction(Call(adapter.callee.as_u32())); let mut result_locals = Vec::with_capacity(lift_sig.results.len()); let mut temps = Vec::new(); for ty in lift_sig.results.iter().rev() { let local = self.local_set_new_tmp(*ty); result_locals.push((local.idx, *ty)); temps.push(local); } result_locals.reverse(); // Like above during the translation of results the caller cannot be // left (as we might invoke things like `realloc`). Again the precise // order of everything doesn't matter since intermediate states cannot // be witnessed, hence the setting of flags here to encapsulate both // liftings and lowerings. // // TODO: like above the management of the `MAY_LEAVE` flag can probably // be elided here for "simple" results. self.set_flag(adapter.lower.flags, FLAG_MAY_LEAVE, false); self.translate_results(adapter, ¶m_locals, &result_locals); self.set_flag(adapter.lower.flags, FLAG_MAY_LEAVE, true); // And finally post-return state is handled here once all results/etc // are all translated. if let Some(func) = adapter.lift.post_return { for (result, _) in result_locals.iter() { self.instruction(LocalGet(*result)); } self.instruction(Call(func.as_u32())); } if adapter.called_as_export { self.set_flag(adapter.lift.flags, FLAG_MAY_ENTER, true); } for tmp in temps { self.free_temp_local(tmp); } if self.emit_resource_call { let exit = self.module.import_resource_exit_call(); self.instruction(Call(exit.as_u32())); } self.finish() } fn translate_params(&mut self, adapter: &AdapterData, param_locals: &[(u32, ValType)]) { let src_tys = self.types[adapter.lower.ty].params; let src_tys = self.types[src_tys] .types .iter() .copied() .collect::<Vec<_>>(); let dst_tys = self.types[adapter.lift.ty].params; let dst_tys = self.types[dst_tys] .types .iter() .copied() .collect::<Vec<_>>(); let lift_opts = &adapter.lift.options; let lower_opts = &adapter.lower.options; // TODO: handle subtyping assert_eq!(src_tys.len(), dst_tys.len()); let src_flat = self.types .flatten_types(lower_opts, MAX_FLAT_PARAMS, src_tys.iter().copied()); let dst_flat = self.types .flatten_types(lift_opts, MAX_FLAT_PARAMS, dst_tys.iter().copied()); let src = if let Some(flat) = &src_flat { Source::Stack(Stack { locals: ¶m_locals[..flat.len()], opts: lower_opts, }) } else { // If there are too many parameters then that means the parameters // are actually a tuple stored in linear memory addressed by the // first parameter local. let (addr, ty) = param_locals[0]; assert_eq!(ty, lower_opts.ptr()); let align = src_tys .iter() .map(|t| self.types.align(lower_opts, t)) .max() .unwrap_or(1); Source::Memory(self.memory_operand(lower_opts, TempLocal::new(addr, ty), align)) }; let dst = if let Some(flat) = &dst_flat { Destination::Stack(flat, lift_opts) } else { // If there are too many parameters then space is allocated in the // destination module for the parameters via its `realloc` function. let abi = CanonicalAbiInfo::record(dst_tys.iter().map(|t| self.types.canonical_abi(t))); let (size, align) = if lift_opts.memory64 { (abi.size64, abi.align64) } else { (abi.size32, abi.align32) }; let size = MallocSize::Const(size); Destination::Memory(self.malloc(lift_opts, size, align)) }; let srcs = src .record_field_srcs(self.types, src_tys.iter().copied()) .zip(src_tys.iter()); let dsts = dst .record_field_dsts(self.types, dst_tys.iter().copied()) .zip(dst_tys.iter()); for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) { self.translate(&src_ty, &src, &dst_ty, &dst); } // If the destination was linear memory instead of the stack then the // actual parameter that we're passing is the address of the values // stored, so ensure that's happening in the wasm body here. if let Destination::Memory(mem) = dst { self.instruction(LocalGet(mem.addr.idx)); self.free_temp_local(mem.addr); } } fn translate_results( &mut self, adapter: &AdapterData, param_locals: &[(u32, ValType)], result_locals: &[(u32, ValType)], ) { let src_tys = self.types[adapter.lift.ty].results; let src_tys = self.types[src_tys] .types .iter() .copied() .collect::<Vec<_>>(); let dst_tys = self.types[adapter.lower.ty].results; let dst_tys = self.types[dst_tys] .types .iter() .copied() .collect::<Vec<_>>(); let lift_opts = &adapter.lift.options; let lower_opts = &adapter.lower.options; let src_flat = self.types .flatten_types(lift_opts, MAX_FLAT_RESULTS, src_tys.iter().copied()); let dst_flat = self.types .flatten_types(lower_opts, MAX_FLAT_RESULTS, dst_tys.iter().copied()); let src = if src_flat.is_some() { Source::Stack(Stack { locals: result_locals, opts: lift_opts, }) } else { // The original results to read from in this case come from the // return value of the function itself. The imported function will // return a linear memory address at which the values can be read // from. let align = src_tys .iter() .map(|t| self.types.align(lift_opts, t)) .max() .unwrap_or(1); assert_eq!(result_locals.len(), 1); let (addr, ty) = result_locals[0]; assert_eq!(ty, lift_opts.ptr()); Source::Memory(self.memory_operand(lift_opts, TempLocal::new(addr, ty), align)) }; let dst = if let Some(flat) = &dst_flat { Destination::Stack(flat, lower_opts) } else { // This is slightly different than `translate_params` where the // return pointer was provided by the caller of this function // meaning the last parameter local is a pointer into linear memory. let align = dst_tys .iter() .map(|t| self.types.align(lower_opts, t)) .max() .unwrap_or(1); let (addr, ty) = *param_locals.last().expect("no retptr"); assert_eq!(ty, lower_opts.ptr()); Destination::Memory(self.memory_operand(lower_opts, TempLocal::new(addr, ty), align)) }; let srcs = src .record_field_srcs(self.types, src_tys.iter().copied()) .zip(src_tys.iter()); let dsts = dst .record_field_dsts(self.types, dst_tys.iter().copied()) .zip(dst_tys.iter()); for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) { self.translate(&src_ty, &src, &dst_ty, &dst); } } fn translate( &mut self, src_ty: &InterfaceType, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { if let Source::Memory(mem) = src { self.assert_aligned(src_ty, mem); } if let Destination::Memory(mem) = dst { self.assert_aligned(dst_ty, mem); } // Calculate a cost heuristic for what the translation of this specific // layer of the type is going to incur. The purpose of this cost is that // we'll deduct it from `self.fuel` and if no fuel is remaining then // translation is outlined into a separate function rather than being // translated into this function. // // The general goal is to avoid creating an exponentially sized function // for a linearly sized input (the type section). By outlining helper // functions there will ideally be a constant set of helper functions // per type (to accommodate in-memory or on-stack transfers as well as // src/dst options) which means that each function is at most a certain // size and we have a linear number of functions which should guarantee // an overall linear size of the output. // // To implement this the current heuristic is that each layer of // translating a type has a cost associated with it and this cost is // accounted for in `self.fuel`. Some conversions are considered free as // they generate basically as much code as the `call` to the translation // function while other are considered proportionally expensive to the // size of the type. The hope is that some upper layers are of a type's // translation are all inlined into one function but bottom layers end // up getting outlined to separate functions. Theoretically, again this // is built on hopes and dreams, the outlining can be shared amongst // tightly-intertwined type hierarchies which will reduce the size of // the output module due to the helpers being used. // // This heuristic of how to split functions has changed a few times in // the past and this isn't necessarily guaranteed to be the final // iteration. let cost = match src_ty { // These types are all quite simple to load/store and equate to // basically the same cost of the `call` instruction to call an // out-of-line translation function, so give them 0 cost. InterfaceType::Bool | InterfaceType::U8 | InterfaceType::S8 | InterfaceType::U16 | InterfaceType::S16 | InterfaceType::U32 | InterfaceType::S32 | InterfaceType::U64 | InterfaceType::S64 | InterfaceType::Float32 | InterfaceType::Float64 => 0, // This has a small amount of validation associated with it, so // give it a cost of 1. InterfaceType::Char => 1, // This has a fair bit of code behind it depending on the // strings/encodings in play, so arbitrarily assign it this cost. InterfaceType::String => 40, // Iteration of a loop is along the lines of the cost of a string // so give it the same cost InterfaceType::List(_) => 40, InterfaceType::Flags(i) => { let count = self.module.types[*i].names.len(); match FlagsSize::from_count(count) { FlagsSize::Size0 => 0, FlagsSize::Size1 | FlagsSize::Size2 => 1, FlagsSize::Size4Plus(n) => n.into(), } } InterfaceType::Record(i) => self.types[*i].fields.len(), InterfaceType::Tuple(i) => self.types[*i].types.len(), InterfaceType::Variant(i) => self.types[*i].cases.len(), InterfaceType::Enum(i) => self.types[*i].names.len(), // 2 cases to consider for each of these variants. InterfaceType::Option(_) | InterfaceType::Result(_) => 2, // TODO(#6696) - something nonzero, is 1 right? InterfaceType::Own(_) | InterfaceType::Borrow(_) => 1, }; match self.fuel.checked_sub(cost) { // This function has enough fuel to perform the layer of translation // necessary for this type, so the fuel is updated in-place and // translation continues. Note that the recursion here is bounded by // the static recursion limit for all interface types as imposed // during the translation phase. Some(n) => { self.fuel = n; match src_ty { InterfaceType::Bool => self.translate_bool(src, dst_ty, dst), InterfaceType::U8 => self.translate_u8(src, dst_ty, dst), InterfaceType::S8 => self.translate_s8(src, dst_ty, dst), InterfaceType::U16 => self.translate_u16(src, dst_ty, dst), InterfaceType::S16 => self.translate_s16(src, dst_ty, dst), InterfaceType::U32 => self.translate_u32(src, dst_ty, dst), InterfaceType::S32 => self.translate_s32(src, dst_ty, dst), InterfaceType::U64 => self.translate_u64(src, dst_ty, dst), InterfaceType::S64 => self.translate_s64(src, dst_ty, dst), InterfaceType::Float32 => self.translate_f32(src, dst_ty, dst), InterfaceType::Float64 => self.translate_f64(src, dst_ty, dst), InterfaceType::Char => self.translate_char(src, dst_ty, dst), InterfaceType::String => self.translate_string(src, dst_ty, dst), InterfaceType::List(t) => self.translate_list(*t, src, dst_ty, dst), InterfaceType::Record(t) => self.translate_record(*t, src, dst_ty, dst), InterfaceType::Flags(f) => self.translate_flags(*f, src, dst_ty, dst), InterfaceType::Tuple(t) => self.translate_tuple(*t, src, dst_ty, dst), InterfaceType::Variant(v) => self.translate_variant(*v, src, dst_ty, dst), InterfaceType::Enum(t) => self.translate_enum(*t, src, dst_ty, dst), InterfaceType::Option(t) => self.translate_option(*t, src, dst_ty, dst), InterfaceType::Result(t) => self.translate_result(*t, src, dst_ty, dst), InterfaceType::Own(t) => self.translate_own(*t, src, dst_ty, dst), InterfaceType::Borrow(t) => self.translate_borrow(*t, src, dst_ty, dst), } } // This function does not have enough fuel left to perform this // layer of translation so the translation is deferred to a helper // function. The actual translation here is then done by marshalling // the src/dst into the function we're calling and then processing // the results. None => { let src_loc = match src { // If the source is on the stack then `stack_get` is used to // convert everything to the appropriate flat representation // for the source type. Source::Stack(stack) => { for (i, ty) in stack .opts .flat_types(src_ty, self.types) .unwrap() .iter() .enumerate() { let stack = stack.slice(i..i + 1); self.stack_get(&stack, (*ty).into()); } HelperLocation::Stack } // If the source is in memory then the pointer is passed // through, but note that the offset must be factored in // here since the translation function will start from // offset 0. Source::Memory(mem) => { self.push_mem_addr(mem); HelperLocation::Memory } }; let dst_loc = match dst { Destination::Stack(..) => HelperLocation::Stack, Destination::Memory(mem) => { self.push_mem_addr(mem); HelperLocation::Memory } }; // Generate a `FunctionId` corresponding to the `Helper` // configuration that is necessary here. This will ideally be a // "cache hit" and use a preexisting helper which represents // outlining what would otherwise be duplicate code within a // function to one function. let helper = self.module.translate_helper(Helper { src: HelperType { ty: *src_ty, opts: *src.opts(), loc: src_loc, }, dst: HelperType { ty: *dst_ty, opts: *dst.opts(), loc: dst_loc, }, }); // Emit a `call` instruction which will get "relocated" to a // function index once translation has completely finished. self.flush_code(); self.module.funcs[self.result].body.push(Body::Call(helper)); // If the destination of the translation was on the stack then // the types on the stack need to be optionally converted to // different types (e.g. if the result here is part of a variant // somewhere else). // // This translation happens inline here by popping the results // into new locals and then using those locals to do a // `stack_set`. if let Destination::Stack(tys, opts) = dst { let flat = self .types .flatten_types(opts, usize::MAX, [*dst_ty]) .unwrap(); assert_eq!(flat.len(), tys.len()); let locals = flat .iter() .rev() .map(|ty| self.local_set_new_tmp(*ty)) .collect::<Vec<_>>(); for (ty, local) in tys.iter().zip(locals.into_iter().rev()) { self.instruction(LocalGet(local.idx)); self.stack_set(std::slice::from_ref(ty), local.ty); self.free_temp_local(local); } } } } } fn push_mem_addr(&mut self, mem: &Memory<'_>) { self.instruction(LocalGet(mem.addr.idx)); if mem.offset != 0 { self.ptr_uconst(mem.opts, mem.offset); self.ptr_add(mem.opts); } } fn translate_bool(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::Bool)); self.push_dst_addr(dst); // Booleans are canonicalized to 0 or 1 as they pass through the // component boundary, so use a `select` instruction to do so. self.instruction(I32Const(1)); self.instruction(I32Const(0)); match src { Source::Memory(mem) => self.i32_load8u(mem), Source::Stack(stack) => self.stack_get(stack, ValType::I32), } self.instruction(Select); match dst { Destination::Memory(mem) => self.i32_store8(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } } fn translate_u8(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::U8)); self.convert_u8_mask(src, dst, 0xff); } fn convert_u8_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u8) { self.push_dst_addr(dst); let mut needs_mask = true; match src { Source::Memory(mem) => { self.i32_load8u(mem); needs_mask = mask != 0xff; } Source::Stack(stack) => { self.stack_get(stack, ValType::I32); } } if needs_mask { self.instruction(I32Const(i32::from(mask))); self.instruction(I32And); } match dst { Destination::Memory(mem) => self.i32_store8(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } } fn translate_s8(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::S8)); self.push_dst_addr(dst); match src { Source::Memory(mem) => self.i32_load8s(mem), Source::Stack(stack) => { self.stack_get(stack, ValType::I32); self.instruction(I32Extend8S); } } match dst { Destination::Memory(mem) => self.i32_store8(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } } fn translate_u16(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::U16)); self.convert_u16_mask(src, dst, 0xffff); } fn convert_u16_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u16) { self.push_dst_addr(dst); let mut needs_mask = true; match src { Source::Memory(mem) => { self.i32_load16u(mem); needs_mask = mask != 0xffff; } Source::Stack(stack) => { self.stack_get(stack, ValType::I32); } } if needs_mask { self.instruction(I32Const(i32::from(mask))); self.instruction(I32And); } match dst { Destination::Memory(mem) => self.i32_store16(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } } fn translate_s16(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::S16)); self.push_dst_addr(dst); match src { Source::Memory(mem) => self.i32_load16s(mem), Source::Stack(stack) => { self.stack_get(stack, ValType::I32); self.instruction(I32Extend16S); } } match dst { Destination::Memory(mem) => self.i32_store16(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } } fn translate_u32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::U32)); self.convert_u32_mask(src, dst, 0xffffffff) } fn convert_u32_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u32) { self.push_dst_addr(dst); match src { Source::Memory(mem) => self.i32_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::I32), } if mask != 0xffffffff { self.instruction(I32Const(mask as i32)); self.instruction(I32And); } match dst { Destination::Memory(mem) => self.i32_store(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } } fn translate_s32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::S32)); self.push_dst_addr(dst); match src { Source::Memory(mem) => self.i32_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::I32), } match dst { Destination::Memory(mem) => self.i32_store(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } } fn translate_u64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::U64)); self.push_dst_addr(dst); match src { Source::Memory(mem) => self.i64_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::I64), } match dst { Destination::Memory(mem) => self.i64_store(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I64), } } fn translate_s64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::S64)); self.push_dst_addr(dst); match src { Source::Memory(mem) => self.i64_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::I64), } match dst { Destination::Memory(mem) => self.i64_store(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I64), } } fn translate_f32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::Float32)); self.push_dst_addr(dst); match src { Source::Memory(mem) => self.f32_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::F32), } match dst { Destination::Memory(mem) => self.f32_store(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::F32), } } fn translate_f64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { // TODO: subtyping assert!(matches!(dst_ty, InterfaceType::Float64)); self.push_dst_addr(dst); match src { Source::Memory(mem) => self.f64_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::F64), } match dst { Destination::Memory(mem) => self.f64_store(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::F64), } } fn translate_char(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { assert!(matches!(dst_ty, InterfaceType::Char)); match src { Source::Memory(mem) => self.i32_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::I32), } let local = self.local_set_new_tmp(ValType::I32); // This sequence is copied from the output of LLVM for: // // pub extern "C" fn foo(x: u32) -> char { // char::try_from(x) // .unwrap_or_else(|_| std::arch::wasm32::unreachable()) // } // // Apparently this does what's required by the canonical ABI: // // def i32_to_char(opts, i): // trap_if(i >= 0x110000) // trap_if(0xD800 <= i <= 0xDFFF) // return chr(i) // // ... but I don't know how it works other than "well I trust LLVM" self.instruction(Block(BlockType::Empty)); self.instruction(Block(BlockType::Empty)); self.instruction(LocalGet(local.idx)); self.instruction(I32Const(0xd800)); self.instruction(I32Xor); self.instruction(I32Const(-0x110000)); self.instruction(I32Add); self.instruction(I32Const(-0x10f800)); self.instruction(I32LtU); self.instruction(BrIf(0)); self.instruction(LocalGet(local.idx)); self.instruction(I32Const(0x110000)); self.instruction(I32Ne); self.instruction(BrIf(1)); self.instruction(End); self.trap(Trap::InvalidChar); self.instruction(End); self.push_dst_addr(dst); self.instruction(LocalGet(local.idx)); match dst { Destination::Memory(mem) => { self.i32_store(mem); } Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } self.free_temp_local(local); } fn translate_string(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { assert!(matches!(dst_ty, InterfaceType::String)); let src_opts = src.opts(); let dst_opts = dst.opts(); // Load the pointer/length of this string into temporary locals. These // will be referenced a good deal so this just makes it easier to deal // with them consistently below rather than trying to reload from memory // for example. match src { Source::Stack(s) => { assert_eq!(s.locals.len(), 2); self.stack_get(&s.slice(0..1), src_opts.ptr()); self.stack_get(&s.slice(1..2), src_opts.ptr()); } Source::Memory(mem) => { self.ptr_load(mem); self.ptr_load(&mem.bump(src_opts.ptr_size().into())); } } let src_len = self.local_set_new_tmp(src_opts.ptr()); let src_ptr = self.local_set_new_tmp(src_opts.ptr()); let src_str = WasmString { ptr: src_ptr, len: src_len, opts: src_opts, }; let dst_str = match src_opts.string_encoding { StringEncoding::Utf8 => match dst_opts.string_encoding { StringEncoding::Utf8 => self.string_copy(&src_str, FE::Utf8, dst_opts, FE::Utf8), StringEncoding::Utf16 => self.string_utf8_to_utf16(&src_str, dst_opts), StringEncoding::CompactUtf16 => { self.string_to_compact(&src_str, FE::Utf8, dst_opts) } }, StringEncoding::Utf16 => { self.verify_aligned(src_opts, src_str.ptr.idx, 2); match dst_opts.string_encoding { StringEncoding::Utf8 => { self.string_deflate_to_utf8(&src_str, FE::Utf16, dst_opts) } StringEncoding::Utf16 => { self.string_copy(&src_str, FE::Utf16, dst_opts, FE::Utf16) } StringEncoding::CompactUtf16 => { self.string_to_compact(&src_str, FE::Utf16, dst_opts) } } } StringEncoding::CompactUtf16 => { self.verify_aligned(src_opts, src_str.ptr.idx, 2); // Test the tag big to see if this is a utf16 or a latin1 string // at runtime... self.instruction(LocalGet(src_str.len.idx)); self.ptr_uconst(src_opts, UTF16_TAG); self.ptr_and(src_opts); self.ptr_if(src_opts, BlockType::Empty); // In the utf16 block unset the upper bit from the length local // so further calculations have the right value. Afterwards the // string transcode proceeds assuming utf16. self.instruction(LocalGet(src_str.len.idx)); self.ptr_uconst(src_opts, UTF16_TAG); self.ptr_xor(src_opts); self.instruction(LocalSet(src_str.len.idx)); let s1 = match dst_opts.string_encoding { StringEncoding::Utf8 => { self.string_deflate_to_utf8(&src_str, FE::Utf16, dst_opts) } StringEncoding::Utf16 => { self.string_copy(&src_str, FE::Utf16, dst_opts, FE::Utf16) } StringEncoding::CompactUtf16 => { self.string_compact_utf16_to_compact(&src_str, dst_opts) } }; self.instruction(Else); // In the latin1 block the `src_len` local is already the number // of code units, so the string transcoding is all that needs to // happen. let s2 = match dst_opts.string_encoding { StringEncoding::Utf16 => { self.string_copy(&src_str, FE::Latin1, dst_opts, FE::Utf16) } StringEncoding::Utf8 => { self.string_deflate_to_utf8(&src_str, FE::Latin1, dst_opts) } StringEncoding::CompactUtf16 => { self.string_copy(&src_str, FE::Latin1, dst_opts, FE::Latin1) } }; // Set our `s2` generated locals to the `s2` generated locals // as the resulting pointer of this transcode. self.instruction(LocalGet(s2.ptr.idx)); self.instruction(LocalSet(s1.ptr.idx)); self.instruction(LocalGet(s2.len.idx)); self.instruction(LocalSet(s1.len.idx)); self.instruction(End); self.free_temp_local(s2.ptr); self.free_temp_local(s2.len); s1 } }; // Store the ptr/length in the desired destination match dst { Destination::Stack(s, _) => { self.instruction(LocalGet(dst_str.ptr.idx)); self.stack_set(&s[..1], dst_opts.ptr()); self.instruction(LocalGet(dst_str.len.idx)); self.stack_set(&s[1..], dst_opts.ptr()); } Destination::Memory(mem) => { self.instruction(LocalGet(mem.addr.idx)); self.instruction(LocalGet(dst_str.ptr.idx)); self.ptr_store(mem); self.instruction(LocalGet(mem.addr.idx)); self.instruction(LocalGet(dst_str.len.idx)); self.ptr_store(&mem.bump(dst_opts.ptr_size().into())); } } self.free_temp_local(src_str.ptr); self.free_temp_local(src_str.len); self.free_temp_local(dst_str.ptr); self.free_temp_local(dst_str.len); } // Corresponding function for `store_string_copy` in the spec. // // This performs a transcoding of the string with a one-pass copy from // the `src` encoding to the `dst` encoding. This is only possible for // fixed encodings where the first allocation is guaranteed to be an // appropriate fit so it's not suitable for all encodings. // // Imported host transcoding functions here take the src/dst pointers as // well as the number of code units in the source (which always matches // the number of code units in the destination). There is no return // value from the transcode function since the encoding should always // work on the first pass. fn string_copy<'a>( &mut self, src: &WasmString<'_>, src_enc: FE, dst_opts: &'a Options, dst_enc: FE, ) -> WasmString<'a> { assert!(dst_enc.width() >= src_enc.width()); self.validate_string_length(src, dst_enc); // Calculate the source byte length given the size of each code // unit. Note that this shouldn't overflow given // `validate_string_length` above. let mut src_byte_len_tmp = None; let src_byte_len = if src_enc.width() == 1 { src.len.idx } else { assert_eq!(src_enc.width(), 2); self.instruction(LocalGet(src.len.idx)); self.ptr_uconst(src.opts, 1); self.ptr_shl(src.opts); let tmp = self.local_set_new_tmp(src.opts.ptr()); let ret = tmp.idx; src_byte_len_tmp = Some(tmp); ret }; // Convert the source code units length to the destination byte // length type. self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); let dst_len = self.local_tee_new_tmp(dst_opts.ptr()); if dst_enc.width() > 1 { assert_eq!(dst_enc.width(), 2); self.ptr_uconst(dst_opts, 1); self.ptr_shl(dst_opts); } let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr()); // Allocate space in the destination using the calculated byte // length. let dst = { let dst_mem = self.malloc( dst_opts, MallocSize::Local(dst_byte_len.idx), dst_enc.width().into(), ); WasmString { ptr: dst_mem.addr, len: dst_len, opts: dst_opts, } }; // Validate that `src_len + src_ptr` and // `dst_mem.addr_local + dst_byte_len` are both in-bounds. This // is done by loading the last byte of the string and if that // doesn't trap then it's known valid. self.validate_string_inbounds(src, src_byte_len); self.validate_string_inbounds(&dst, dst_byte_len.idx); // If the validations pass then the host `transcode` intrinsic // is invoked. This will either raise a trap or otherwise succeed // in which case we're done. let op = if src_enc == dst_enc { Transcode::Copy(src_enc) } else { assert_eq!(src_enc, FE::Latin1); assert_eq!(dst_enc, FE::Utf16); Transcode::Latin1ToUtf16 }; let transcode = self.transcoder(src, &dst, op); self.instruction(LocalGet(src.ptr.idx)); self.instruction(LocalGet(src.len.idx)); self.instruction(LocalGet(dst.ptr.idx)); self.instruction(Call(transcode.as_u32())); self.free_temp_local(dst_byte_len); if let Some(tmp) = src_byte_len_tmp { self.free_temp_local(tmp); } dst } // Corresponding function for `store_string_to_utf8` in the spec. // // This translation works by possibly performing a number of // reallocations. First a buffer of size input-code-units is used to try // to get the transcoding correct on the first try. If that fails the // maximum worst-case size is used and then that is resized down if it's // too large. // // The host transcoding function imported here will receive src ptr/len // and dst ptr/len and return how many code units were consumed on both // sides. The amount of code units consumed in the source dictates which // branches are taken in this conversion. fn string_deflate_to_utf8<'a>( &mut self, src: &WasmString<'_>, src_enc: FE, dst_opts: &'a Options, ) -> WasmString<'a> { self.validate_string_length(src, src_enc); // Optimistically assume that the code unit length of the source is // all that's needed in the destination. Perform that allocation // here and proceed to transcoding below. self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); let dst_len = self.local_tee_new_tmp(dst_opts.ptr()); let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr()); let dst = { let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 1); WasmString { ptr: dst_mem.addr, len: dst_len, opts: dst_opts, } }; // Ensure buffers are all in-bounds let mut src_byte_len_tmp = None; let src_byte_len = match src_enc { FE::Latin1 => src.len.idx, FE::Utf16 => { self.instruction(LocalGet(src.len.idx)); self.ptr_uconst(src.opts, 1); self.ptr_shl(src.opts); let tmp = self.local_set_new_tmp(src.opts.ptr()); let ret = tmp.idx; src_byte_len_tmp = Some(tmp); ret } FE::Utf8 => unreachable!(), }; self.validate_string_inbounds(src, src_byte_len); self.validate_string_inbounds(&dst, dst_byte_len.idx); // Perform the initial transcode let op = match src_enc { FE::Latin1 => Transcode::Latin1ToUtf8, FE::Utf16 => Transcode::Utf16ToUtf8, FE::Utf8 => unreachable!(), }; let transcode = self.transcoder(src, &dst, op); self.instruction(LocalGet(src.ptr.idx)); self.instruction(LocalGet(src.len.idx)); self.instruction(LocalGet(dst.ptr.idx)); self.instruction(LocalGet(dst_byte_len.idx)); self.instruction(Call(transcode.as_u32())); self.instruction(LocalSet(dst.len.idx)); let src_len_tmp = self.local_set_new_tmp(src.opts.ptr()); // Test if the source was entirely transcoded by comparing // `src_len_tmp`, the number of code units transcoded from the // source, with `src_len`, the original number of code units. self.instruction(LocalGet(src_len_tmp.idx)); self.instruction(LocalGet(src.len.idx)); self.ptr_ne(src.opts); self.instruction(If(BlockType::Empty)); // Here a worst-case reallocation is performed to grow `dst_mem`. // In-line a check is also performed that the worst-case byte size // fits within the maximum size of strings. self.instruction(LocalGet(dst.ptr.idx)); // old_ptr self.instruction(LocalGet(dst_byte_len.idx)); // old_size self.ptr_uconst(dst.opts, 1); // align let factor = match src_enc { FE::Latin1 => 2, FE::Utf16 => 3, _ => unreachable!(), }; self.validate_string_length_u8(src, factor); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); self.ptr_uconst(dst_opts, factor.into()); self.ptr_mul(dst_opts); self.instruction(LocalTee(dst_byte_len.idx)); self.instruction(Call(dst_opts.realloc.unwrap().as_u32())); self.instruction(LocalSet(dst.ptr.idx)); // Verify that the destination is still in-bounds self.validate_string_inbounds(&dst, dst_byte_len.idx); // Perform another round of transcoding that should be guaranteed // to succeed. Note that all the parameters here are offset by the // results of the first transcoding to only perform the remaining // transcode on the final units. self.instruction(LocalGet(src.ptr.idx)); self.instruction(LocalGet(src_len_tmp.idx)); if let FE::Utf16 = src_enc { self.ptr_uconst(src.opts, 1); self.ptr_shl(src.opts); } self.ptr_add(src.opts); self.instruction(LocalGet(src.len.idx)); self.instruction(LocalGet(src_len_tmp.idx)); self.ptr_sub(src.opts); self.instruction(LocalGet(dst.ptr.idx)); self.instruction(LocalGet(dst.len.idx)); self.ptr_add(dst.opts); self.instruction(LocalGet(dst_byte_len.idx)); self.instruction(LocalGet(dst.len.idx)); self.ptr_sub(dst.opts); self.instruction(Call(transcode.as_u32())); // Add the second result, the amount of destination units encoded, // to `dst_len` so it's an accurate reflection of the final size of // the destination buffer. self.instruction(LocalGet(dst.len.idx)); self.ptr_add(dst.opts); self.instruction(LocalSet(dst.len.idx)); // In debug mode verify the first result consumed the entire string, // otherwise simply discard it. if self.module.debug { self.instruction(LocalGet(src.len.idx)); self.instruction(LocalGet(src_len_tmp.idx)); self.ptr_sub(src.opts); self.ptr_ne(src.opts); self.instruction(If(BlockType::Empty)); self.trap(Trap::AssertFailed("should have finished encoding")); self.instruction(End); } else { self.instruction(Drop); } // Perform a downsizing if the worst-case size was too large self.instruction(LocalGet(dst.len.idx)); self.instruction(LocalGet(dst_byte_len.idx)); self.ptr_ne(dst.opts); self.instruction(If(BlockType::Empty)); self.instruction(LocalGet(dst.ptr.idx)); // old_ptr self.instruction(LocalGet(dst_byte_len.idx)); // old_size self.ptr_uconst(dst.opts, 1); // align self.instruction(LocalGet(dst.len.idx)); // new_size self.instruction(Call(dst.opts.realloc.unwrap().as_u32())); self.instruction(LocalSet(dst.ptr.idx)); self.instruction(End); // If the first transcode was enough then assert that the returned // amount of destination items written equals the byte size. if self.module.debug { self.instruction(Else); self.instruction(LocalGet(dst.len.idx)); self.instruction(LocalGet(dst_byte_len.idx)); self.ptr_ne(dst_opts); self.instruction(If(BlockType::Empty)); self.trap(Trap::AssertFailed("should have finished encoding")); self.instruction(End); } self.instruction(End); // end of "first transcode not enough" self.free_temp_local(src_len_tmp); self.free_temp_local(dst_byte_len); if let Some(tmp) = src_byte_len_tmp { self.free_temp_local(tmp); } dst } // Corresponds to the `store_utf8_to_utf16` function in the spec. // // When converting utf-8 to utf-16 a pessimistic allocation is // done which is twice the byte length of the utf-8 string. // The host then transcodes and returns how many code units were // actually used during the transcoding and if it's beneath the // pessimistic maximum then the buffer is reallocated down to // a smaller amount. // // The host-imported transcoding function takes the src/dst pointer as // well as the code unit size of both the source and destination. The // destination should always be big enough to hold the result of the // transcode and so the result of the host function is how many code // units were written to the destination. fn string_utf8_to_utf16<'a>( &mut self, src: &WasmString<'_>, dst_opts: &'a Options, ) -> WasmString<'a> { self.validate_string_length(src, FE::Utf16); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); let dst_len = self.local_tee_new_tmp(dst_opts.ptr()); self.ptr_uconst(dst_opts, 1); self.ptr_shl(dst_opts); let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr()); let dst = { let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2); WasmString { ptr: dst_mem.addr, len: dst_len, opts: dst_opts, } }; self.validate_string_inbounds(src, src.len.idx); self.validate_string_inbounds(&dst, dst_byte_len.idx); let transcode = self.transcoder(src, &dst, Transcode::Utf8ToUtf16); self.instruction(LocalGet(src.ptr.idx)); self.instruction(LocalGet(src.len.idx)); self.instruction(LocalGet(dst.ptr.idx)); self.instruction(Call(transcode.as_u32())); self.instruction(LocalSet(dst.len.idx)); // If the number of code units returned by transcode is not // equal to the original number of code units then // the buffer must be shrunk. // // Note that the byte length of the final allocation we // want is twice the code unit length returned by the // transcoding function. self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr()); self.instruction(LocalGet(dst.len.idx)); self.ptr_ne(dst_opts); self.instruction(If(BlockType::Empty)); self.instruction(LocalGet(dst.ptr.idx)); self.instruction(LocalGet(dst_byte_len.idx)); self.ptr_uconst(dst.opts, 2); self.instruction(LocalGet(dst.len.idx)); self.ptr_uconst(dst.opts, 1); self.ptr_shl(dst.opts); self.instruction(Call(dst.opts.realloc.unwrap().as_u32())); self.instruction(LocalSet(dst.ptr.idx)); self.instruction(End); // end of shrink-to-fit self.free_temp_local(dst_byte_len); dst } // Corresponds to `store_probably_utf16_to_latin1_or_utf16` in the spec. // // This will try to transcode the input utf16 string to utf16 in the // destination. If utf16 isn't needed though and latin1 could be used // then that's used instead and a reallocation to downsize occurs // afterwards. // // The host transcode function here will take the src/dst pointers as // well as src length. The destination byte length is twice the src code // unit length. The return value is the tagged length of the returned // string. If the upper bit is set then utf16 was used and the // conversion is done. If the upper bit is not set then latin1 was used // and a downsizing needs to happen. fn string_compact_utf16_to_compact<'a>( &mut self, src: &WasmString<'_>, dst_opts: &'a Options, ) -> WasmString<'a> { self.validate_string_length(src, FE::Utf16); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); let dst_len = self.local_tee_new_tmp(dst_opts.ptr()); self.ptr_uconst(dst_opts, 1); self.ptr_shl(dst_opts); let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr()); let dst = { let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2); WasmString { ptr: dst_mem.addr, len: dst_len, opts: dst_opts, } }; self.convert_src_len_to_dst(dst_byte_len.idx, dst.opts.ptr(), src.opts.ptr()); let src_byte_len = self.local_set_new_tmp(src.opts.ptr()); self.validate_string_inbounds(src, src_byte_len.idx); self.validate_string_inbounds(&dst, dst_byte_len.idx); let transcode = self.transcoder(src, &dst, Transcode::Utf16ToCompactProbablyUtf16); self.instruction(LocalGet(src.ptr.idx)); self.instruction(LocalGet(src.len.idx)); self.instruction(LocalGet(dst.ptr.idx)); self.instruction(Call(transcode.as_u32())); self.instruction(LocalSet(dst.len.idx)); // Assert that the untagged code unit length is the same as the // source code unit length. if self.module.debug { self.instruction(LocalGet(dst.len.idx)); self.ptr_uconst(dst.opts, !UTF16_TAG); self.ptr_and(dst.opts); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr()); self.ptr_ne(dst.opts); self.instruction(If(BlockType::Empty)); self.trap(Trap::AssertFailed("expected equal code units")); self.instruction(End); } // If the UTF16_TAG is set then utf16 was used and the destination // should be appropriately sized. Bail out of the "is this string // empty" block and fall through otherwise to resizing. self.instruction(LocalGet(dst.len.idx)); self.ptr_uconst(dst.opts, UTF16_TAG); self.ptr_and(dst.opts); self.ptr_br_if(dst.opts, 0); // Here `realloc` is used to downsize the string self.instruction(LocalGet(dst.ptr.idx)); // old_ptr self.instruction(LocalGet(dst_byte_len.idx)); // old_size self.ptr_uconst(dst.opts, 2); // align self.instruction(LocalGet(dst.len.idx)); // new_size self.instruction(Call(dst.opts.realloc.unwrap().as_u32())); self.instruction(LocalSet(dst.ptr.idx)); self.free_temp_local(dst_byte_len); self.free_temp_local(src_byte_len); dst } // Corresponds to `store_string_to_latin1_or_utf16` in the spec. // // This will attempt a first pass of transcoding to latin1 and on // failure a larger buffer is allocated for utf16 and then utf16 is // encoded in-place into the buffer. After either latin1 or utf16 the // buffer is then resized to fit the final string allocation. fn string_to_compact<'a>( &mut self, src: &WasmString<'_>, src_enc: FE, dst_opts: &'a Options, ) -> WasmString<'a> { self.validate_string_length(src, src_enc); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); let dst_len = self.local_tee_new_tmp(dst_opts.ptr()); let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr()); let dst = { let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2); WasmString { ptr: dst_mem.addr, len: dst_len, opts: dst_opts, } }; self.validate_string_inbounds(src, src.len.idx); self.validate_string_inbounds(&dst, dst_byte_len.idx); // Perform the initial latin1 transcode. This returns the number of // source code units consumed and the number of destination code // units (bytes) written. let (latin1, utf16) = match src_enc { FE::Utf8 => (Transcode::Utf8ToLatin1, Transcode::Utf8ToCompactUtf16), FE::Utf16 => (Transcode::Utf16ToLatin1, Transcode::Utf16ToCompactUtf16), FE::Latin1 => unreachable!(), }; let transcode_latin1 = self.transcoder(src, &dst, latin1); let transcode_utf16 = self.transcoder(src, &dst, utf16); self.instruction(LocalGet(src.ptr.idx)); self.instruction(LocalGet(src.len.idx)); self.instruction(LocalGet(dst.ptr.idx)); self.instruction(Call(transcode_latin1.as_u32())); self.instruction(LocalSet(dst.len.idx)); let src_len_tmp = self.local_set_new_tmp(src.opts.ptr()); // If the source was entirely consumed then the transcode completed // and all that's necessary is to optionally shrink the buffer. self.instruction(LocalGet(src_len_tmp.idx)); self.instruction(LocalGet(src.len.idx)); self.ptr_eq(src.opts); self.instruction(If(BlockType::Empty)); // if latin1-or-utf16 block // Test if the original byte length of the allocation is the same as // the number of written bytes, and if not then shrink the buffer // with a call to `realloc`. self.instruction(LocalGet(dst_byte_len.idx)); self.instruction(LocalGet(dst.len.idx)); self.ptr_ne(dst.opts); self.instruction(If(BlockType::Empty)); self.instruction(LocalGet(dst.ptr.idx)); // old_ptr self.instruction(LocalGet(dst_byte_len.idx)); // old_size self.ptr_uconst(dst.opts, 2); // align self.instruction(LocalGet(dst.len.idx)); // new_size self.instruction(Call(dst.opts.realloc.unwrap().as_u32())); self.instruction(LocalSet(dst.ptr.idx)); self.instruction(End); // In this block the latin1 encoding failed. The host transcode // returned how many units were consumed from the source and how // many bytes were written to the destination. Here the buffer is // inflated and sized and the second utf16 intrinsic is invoked to // perform the final inflation. self.instruction(Else); // else latin1-or-utf16 block // For utf8 validate that the inflated size is still within bounds. if src_enc.width() == 1 { self.validate_string_length_u8(src, 2); } // Reallocate the buffer with twice the source code units in byte // size. self.instruction(LocalGet(dst.ptr.idx)); // old_ptr self.instruction(LocalGet(dst_byte_len.idx)); // old_size self.ptr_uconst(dst.opts, 2); // align self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr()); self.ptr_uconst(dst.opts, 1); self.ptr_shl(dst.opts); self.instruction(LocalTee(dst_byte_len.idx)); self.instruction(Call(dst.opts.realloc.unwrap().as_u32())); self.instruction(LocalSet(dst.ptr.idx)); // Call the host utf16 transcoding function. This will inflate the // prior latin1 bytes and then encode the rest of the source string // as utf16 into the remaining space in the destination buffer. self.instruction(LocalGet(src.ptr.idx)); self.instruction(LocalGet(src_len_tmp.idx)); if let FE::Utf16 = src_enc { self.ptr_uconst(src.opts, 1); self.ptr_shl(src.opts); } self.ptr_add(src.opts); self.instruction(LocalGet(src.len.idx)); self.instruction(LocalGet(src_len_tmp.idx)); self.ptr_sub(src.opts); self.instruction(LocalGet(dst.ptr.idx)); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr()); self.instruction(LocalGet(dst.len.idx)); self.instruction(Call(transcode_utf16.as_u32())); self.instruction(LocalSet(dst.len.idx)); // If the returned number of code units written to the destination // is not equal to the size of the allocation then the allocation is // resized down to the appropriate size. // // Note that the byte size desired is `2*dst_len` and the current // byte buffer size is `2*src_len` so the `2` factor isn't checked // here, just the lengths. self.instruction(LocalGet(dst.len.idx)); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr()); self.ptr_ne(dst.opts); self.instruction(If(BlockType::Empty)); self.instruction(LocalGet(dst.ptr.idx)); // old_ptr self.instruction(LocalGet(dst_byte_len.idx)); // old_size self.ptr_uconst(dst.opts, 2); // align self.instruction(LocalGet(dst.len.idx)); self.ptr_uconst(dst.opts, 1); self.ptr_shl(dst.opts); self.instruction(Call(dst.opts.realloc.unwrap().as_u32())); self.instruction(LocalSet(dst.ptr.idx)); self.instruction(End); // Tag the returned pointer as utf16 self.instruction(LocalGet(dst.len.idx)); self.ptr_uconst(dst.opts, UTF16_TAG); self.ptr_or(dst.opts); self.instruction(LocalSet(dst.len.idx)); self.instruction(End); // end latin1-or-utf16 block self.free_temp_local(src_len_tmp); self.free_temp_local(dst_byte_len); dst } fn validate_string_length(&mut self, src: &WasmString<'_>, dst: FE) { self.validate_string_length_u8(src, dst.width()) } fn validate_string_length_u8(&mut self, s: &WasmString<'_>, dst: u8) { // Check to see if the source byte length is out of bounds in // which case a trap is generated. self.instruction(LocalGet(s.len.idx)); let max = MAX_STRING_BYTE_LENGTH / u32::from(dst); self.ptr_uconst(s.opts, max); self.ptr_ge_u(s.opts); self.instruction(If(BlockType::Empty)); self.trap(Trap::StringLengthTooBig); self.instruction(End); } fn transcoder( &mut self, src: &WasmString<'_>, dst: &WasmString<'_>, op: Transcode, ) -> FuncIndex { self.module.import_transcoder(Transcoder { from_memory: src.opts.memory.unwrap(), from_memory64: src.opts.memory64, to_memory: dst.opts.memory.unwrap(), to_memory64: dst.opts.memory64, op, }) } fn validate_string_inbounds(&mut self, s: &WasmString<'_>, byte_len: u32) { self.validate_memory_inbounds(s.opts, s.ptr.idx, byte_len, Trap::StringLengthOverflow) } fn validate_memory_inbounds( &mut self, opts: &Options, ptr_local: u32, byte_len_local: u32, trap: Trap, ) { let extend_to_64 = |me: &mut Self| { if !opts.memory64 { me.instruction(I64ExtendI32U); } }; self.instruction(Block(BlockType::Empty)); self.instruction(Block(BlockType::Empty)); // Calculate the full byte size of memory with `memory.size`. Note that // arithmetic here is done always in 64-bits to accommodate 4G memories. // Additionally it's assumed that 64-bit memories never fill up // entirely. self.instruction(MemorySize(opts.memory.unwrap().as_u32())); extend_to_64(self); self.instruction(I64Const(16)); self.instruction(I64Shl); // Calculate the end address of the string. This is done by adding the // base pointer to the byte length. For 32-bit memories there's no need // to check for overflow since everything is extended to 64-bit, but for // 64-bit memories overflow is checked. self.instruction(LocalGet(ptr_local)); extend_to_64(self); self.instruction(LocalGet(byte_len_local)); extend_to_64(self); self.instruction(I64Add); if opts.memory64 { let tmp = self.local_tee_new_tmp(ValType::I64); self.instruction(LocalGet(ptr_local)); self.ptr_lt_u(opts); self.instruction(BrIf(0)); self.instruction(LocalGet(tmp.idx)); self.free_temp_local(tmp); } // If the byte size of memory is greater than the final address of the // string then the string is invalid. Note that if it's precisely equal // then that's ok. self.instruction(I64GeU); self.instruction(BrIf(1)); self.instruction(End); self.trap(trap); self.instruction(End); } fn translate_list( &mut self, src_ty: TypeListIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let src_element_ty = &self.types[src_ty].element; let dst_element_ty = match dst_ty { InterfaceType::List(r) => &self.types[*r].element, _ => panic!("expected a list"), }; let src_opts = src.opts(); let dst_opts = dst.opts(); let (src_size, src_align) = self.types.size_align(src_opts, src_element_ty); let (dst_size, dst_align) = self.types.size_align(dst_opts, dst_element_ty); // Load the pointer/length of this list into temporary locals. These // will be referenced a good deal so this just makes it easier to deal // with them consistently below rather than trying to reload from memory // for example. match src { Source::Stack(s) => { assert_eq!(s.locals.len(), 2); self.stack_get(&s.slice(0..1), src_opts.ptr()); self.stack_get(&s.slice(1..2), src_opts.ptr()); } Source::Memory(mem) => { self.ptr_load(mem); self.ptr_load(&mem.bump(src_opts.ptr_size().into())); } } let src_len = self.local_set_new_tmp(src_opts.ptr()); let src_ptr = self.local_set_new_tmp(src_opts.ptr()); // Create a `Memory` operand which will internally assert that the // `src_ptr` value is properly aligned. let src_mem = self.memory_operand(src_opts, src_ptr, src_align); // Calculate the source/destination byte lengths into unique locals. let src_byte_len = self.calculate_list_byte_len(src_opts, src_len.idx, src_size); let dst_byte_len = if src_size == dst_size { self.convert_src_len_to_dst(src_byte_len.idx, src_opts.ptr(), dst_opts.ptr()); self.local_set_new_tmp(dst_opts.ptr()) } else if src_opts.ptr() == dst_opts.ptr() { self.calculate_list_byte_len(dst_opts, src_len.idx, dst_size) } else { self.convert_src_len_to_dst(src_byte_len.idx, src_opts.ptr(), dst_opts.ptr()); let tmp = self.local_set_new_tmp(dst_opts.ptr()); let ret = self.calculate_list_byte_len(dst_opts, tmp.idx, dst_size); self.free_temp_local(tmp); ret }; // Here `realloc` is invoked (in a `malloc`-like fashion) to allocate // space for the list in the destination memory. This will also // internally insert checks that the returned pointer is aligned // correctly for the destination. let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), dst_align); // With all the pointers and byte lengths verity that both the source // and the destination buffers are in-bounds. self.validate_memory_inbounds( src_opts, src_mem.addr.idx, src_byte_len.idx, Trap::ListByteLengthOverflow, ); self.validate_memory_inbounds( dst_opts, dst_mem.addr.idx, dst_byte_len.idx, Trap::ListByteLengthOverflow, ); self.free_temp_local(src_byte_len); self.free_temp_local(dst_byte_len); // This is the main body of the loop to actually translate list types. // Note that if both element sizes are 0 then this won't actually do // anything so the loop is removed entirely. if src_size > 0 || dst_size > 0 { // This block encompasses the entire loop and is use to exit before even // entering the loop if the list size is zero. self.instruction(Block(BlockType::Empty)); // Set the `remaining` local and only continue if it's > 0 self.instruction(LocalGet(src_len.idx)); let remaining = self.local_tee_new_tmp(src_opts.ptr()); self.ptr_eqz(src_opts); self.instruction(BrIf(0)); // Initialize the two destination pointers to their initial values self.instruction(LocalGet(src_mem.addr.idx)); let cur_src_ptr = self.local_set_new_tmp(src_opts.ptr()); self.instruction(LocalGet(dst_mem.addr.idx)); let cur_dst_ptr = self.local_set_new_tmp(dst_opts.ptr()); self.instruction(Loop(BlockType::Empty)); // Translate the next element in the list let element_src = Source::Memory(Memory { opts: src_opts, offset: 0, addr: TempLocal::new(cur_src_ptr.idx, cur_src_ptr.ty), }); let element_dst = Destination::Memory(Memory { opts: dst_opts, offset: 0, addr: TempLocal::new(cur_dst_ptr.idx, cur_dst_ptr.ty), }); self.translate(src_element_ty, &element_src, dst_element_ty, &element_dst); // Update the two loop pointers if src_size > 0 { self.instruction(LocalGet(cur_src_ptr.idx)); self.ptr_uconst(src_opts, src_size); self.ptr_add(src_opts); self.instruction(LocalSet(cur_src_ptr.idx)); } if dst_size > 0 { self.instruction(LocalGet(cur_dst_ptr.idx)); self.ptr_uconst(dst_opts, dst_size); self.ptr_add(dst_opts); self.instruction(LocalSet(cur_dst_ptr.idx)); } // Update the remaining count, falling through to break out if it's zero // now. self.instruction(LocalGet(remaining.idx)); self.ptr_iconst(src_opts, -1); self.ptr_add(src_opts); self.instruction(LocalTee(remaining.idx)); self.ptr_br_if(src_opts, 0); self.instruction(End); // end of loop self.instruction(End); // end of block self.free_temp_local(cur_dst_ptr); self.free_temp_local(cur_src_ptr); self.free_temp_local(remaining); } // Store the ptr/length in the desired destination match dst { Destination::Stack(s, _) => { self.instruction(LocalGet(dst_mem.addr.idx)); self.stack_set(&s[..1], dst_opts.ptr()); self.convert_src_len_to_dst(src_len.idx, src_opts.ptr(), dst_opts.ptr()); self.stack_set(&s[1..], dst_opts.ptr()); } Destination::Memory(mem) => { self.instruction(LocalGet(mem.addr.idx)); self.instruction(LocalGet(dst_mem.addr.idx)); self.ptr_store(mem); self.instruction(LocalGet(mem.addr.idx)); self.convert_src_len_to_dst(src_len.idx, src_opts.ptr(), dst_opts.ptr()); self.ptr_store(&mem.bump(dst_opts.ptr_size().into())); } } self.free_temp_local(src_len); self.free_temp_local(src_mem.addr); self.free_temp_local(dst_mem.addr); } fn calculate_list_byte_len( &mut self, opts: &Options, len_local: u32, elt_size: u32, ) -> TempLocal { // Zero-size types are easy to handle here because the byte size of the // destination is always zero. if elt_size == 0 { self.ptr_uconst(opts, 0); return self.local_set_new_tmp(opts.ptr()); } // For one-byte elements in the destination the check here can be a bit // more optimal than the general case below. In these situations if the // source pointer type is 32-bit then we're guaranteed to not overflow, // so the source length is simply casted to the destination's type. // // If the source is 64-bit then all that needs to be checked is to // ensure that it does not have the upper 32-bits set. if elt_size == 1 { if let ValType::I64 = opts.ptr() { self.instruction(LocalGet(len_local)); self.instruction(I64Const(32)); self.instruction(I64ShrU); self.instruction(I32WrapI64); self.instruction(If(BlockType::Empty)); self.trap(Trap::ListByteLengthOverflow); self.instruction(End); } self.instruction(LocalGet(len_local)); return self.local_set_new_tmp(opts.ptr()); } // The main check implemented by this function is to verify that // `src_len_local` does not exceed the 32-bit range. Byte sizes for // lists must always fit in 32-bits to get transferred to 32-bit // memories. self.instruction(Block(BlockType::Empty)); self.instruction(Block(BlockType::Empty)); self.instruction(LocalGet(len_local)); match opts.ptr() { // The source's list length is guaranteed to be less than 32-bits // so simply extend it up to a 64-bit type for the multiplication // below. ValType::I32 => self.instruction(I64ExtendI32U), // If the source is a 64-bit memory then if the item length doesn't // fit in 32-bits the byte length definitely won't, so generate a // branch to our overflow trap here if any of the upper 32-bits are set. ValType::I64 => { self.instruction(I64Const(32)); self.instruction(I64ShrU); self.instruction(I32WrapI64); self.instruction(BrIf(0)); self.instruction(LocalGet(len_local)); } _ => unreachable!(), } // Next perform a 64-bit multiplication with the element byte size that // is itself guaranteed to fit in 32-bits. The result is then checked // to see if we overflowed the 32-bit space. The two input operands to // the multiplication are guaranteed to be 32-bits at most which means // that this multiplication shouldn't overflow. // // The result of the multiplication is saved into a local as well to // get the result afterwards. self.instruction(I64Const(elt_size.into())); self.instruction(I64Mul); let tmp = self.local_tee_new_tmp(ValType::I64); // Branch to success if the upper 32-bits are zero, otherwise // fall-through to the trap. self.instruction(I64Const(32)); self.instruction(I64ShrU); self.instruction(I64Eqz); self.instruction(BrIf(1)); self.instruction(End); self.trap(Trap::ListByteLengthOverflow); self.instruction(End); // If a fresh local was used to store the result of the multiplication // then convert it down to 32-bits which should be guaranteed to not // lose information at this point. if opts.ptr() == ValType::I64 { tmp } else { self.instruction(LocalGet(tmp.idx)); self.instruction(I32WrapI64); self.free_temp_local(tmp); self.local_set_new_tmp(ValType::I32) } } fn convert_src_len_to_dst( &mut self, src_len_local: u32, src_ptr_ty: ValType, dst_ptr_ty: ValType, ) { self.instruction(LocalGet(src_len_local)); match (src_ptr_ty, dst_ptr_ty) { (ValType::I32, ValType::I64) => self.instruction(I64ExtendI32U), (ValType::I64, ValType::I32) => self.instruction(I32WrapI64), (src, dst) => assert_eq!(src, dst), } } fn translate_record( &mut self, src_ty: TypeRecordIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let src_ty = &self.types[src_ty]; let dst_ty = match dst_ty { InterfaceType::Record(r) => &self.types[*r], _ => panic!("expected a record"), }; // TODO: subtyping assert_eq!(src_ty.fields.len(), dst_ty.fields.len()); // First a map is made of the source fields to where they're coming // from (e.g. which offset or which locals). This map is keyed by the // fields' names let mut src_fields = HashMap::new(); for (i, src) in src .record_field_srcs(self.types, src_ty.fields.iter().map(|f| f.ty)) .enumerate() { let field = &src_ty.fields[i]; src_fields.insert(&field.name, (src, &field.ty)); } // .. and next translation is performed in the order of the destination // fields in case the destination is the stack to ensure that the stack // has the fields all in the right order. // // Note that the lookup in `src_fields` is an infallible lookup which // will panic if the field isn't found. // // TODO: should that lookup be fallible with subtyping? for (i, dst) in dst .record_field_dsts(self.types, dst_ty.fields.iter().map(|f| f.ty)) .enumerate() { let field = &dst_ty.fields[i]; let (src, src_ty) = &src_fields[&field.name]; self.translate(src_ty, src, &field.ty, &dst); } } fn translate_flags( &mut self, src_ty: TypeFlagsIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let src_ty = &self.types[src_ty]; let dst_ty = match dst_ty { InterfaceType::Flags(r) => &self.types[*r], _ => panic!("expected a record"), }; // TODO: subtyping // // Notably this implementation does not support reordering flags from // the source to the destination nor having more flags in the // destination. Currently this is a copy from source to destination // in-bulk. Otherwise reordering indices would have to have some sort of // fancy bit twiddling tricks or something like that. assert_eq!(src_ty.names, dst_ty.names); let cnt = src_ty.names.len(); match FlagsSize::from_count(cnt) { FlagsSize::Size0 => {} FlagsSize::Size1 => { let mask = if cnt == 8 { 0xff } else { (1 << cnt) - 1 }; self.convert_u8_mask(src, dst, mask); } FlagsSize::Size2 => { let mask = if cnt == 16 { 0xffff } else { (1 << cnt) - 1 }; self.convert_u16_mask(src, dst, mask); } FlagsSize::Size4Plus(n) => { let srcs = src.record_field_srcs(self.types, (0..n).map(|_| InterfaceType::U32)); let dsts = dst.record_field_dsts(self.types, (0..n).map(|_| InterfaceType::U32)); let n = usize::from(n); for (i, (src, dst)) in srcs.zip(dsts).enumerate() { let mask = if i == n - 1 && (cnt % 32 != 0) { (1 << (cnt % 32)) - 1 } else { 0xffffffff }; self.convert_u32_mask(&src, &dst, mask); } } } } fn translate_tuple( &mut self, src_ty: TypeTupleIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let src_ty = &self.types[src_ty]; let dst_ty = match dst_ty { InterfaceType::Tuple(t) => &self.types[*t], _ => panic!("expected a tuple"), }; // TODO: subtyping assert_eq!(src_ty.types.len(), dst_ty.types.len()); let srcs = src .record_field_srcs(self.types, src_ty.types.iter().copied()) .zip(src_ty.types.iter()); let dsts = dst .record_field_dsts(self.types, dst_ty.types.iter().copied()) .zip(dst_ty.types.iter()); for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) { self.translate(src_ty, &src, dst_ty, &dst); } } fn translate_variant( &mut self, src_ty: TypeVariantIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let src_ty = &self.types[src_ty]; let dst_ty = match dst_ty { InterfaceType::Variant(t) => &self.types[*t], _ => panic!("expected a variant"), }; let src_info = variant_info(self.types, src_ty.cases.iter().map(|(_, c)| c.as_ref())); let dst_info = variant_info(self.types, dst_ty.cases.iter().map(|(_, c)| c.as_ref())); let iter = src_ty .cases .iter() .enumerate() .map(|(src_i, (src_case, src_case_ty))| { let dst_i = dst_ty .cases .iter() .position(|(c, _)| c == src_case) .unwrap(); let dst_case_ty = &dst_ty.cases[dst_i]; let src_i = u32::try_from(src_i).unwrap(); let dst_i = u32::try_from(dst_i).unwrap(); VariantCase { src_i, src_ty: src_case_ty.as_ref(), dst_i, dst_ty: dst_case_ty.as_ref(), } }); self.convert_variant(src, &src_info, dst, &dst_info, iter); } fn translate_enum( &mut self, src_ty: TypeEnumIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let src_ty = &self.types[src_ty]; let dst_ty = match dst_ty { InterfaceType::Enum(t) => &self.types[*t], _ => panic!("expected an option"), }; let src_info = variant_info(self.types, src_ty.names.iter().map(|_| None)); let dst_info = variant_info(self.types, dst_ty.names.iter().map(|_| None)); self.convert_variant( src, &src_info, dst, &dst_info, src_ty.names.iter().enumerate().map(|(src_i, src_name)| { let dst_i = dst_ty.names.iter().position(|n| n == src_name).unwrap(); let src_i = u32::try_from(src_i).unwrap(); let dst_i = u32::try_from(dst_i).unwrap(); VariantCase { src_i, dst_i, src_ty: None, dst_ty: None, } }), ); } fn translate_option( &mut self, src_ty: TypeOptionIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let src_ty = &self.types[src_ty].ty; let dst_ty = match dst_ty { InterfaceType::Option(t) => &self.types[*t].ty, _ => panic!("expected an option"), }; let src_ty = Some(src_ty); let dst_ty = Some(dst_ty); let src_info = variant_info(self.types, [None, src_ty]); let dst_info = variant_info(self.types, [None, dst_ty]); self.convert_variant( src, &src_info, dst, &dst_info, [ VariantCase { src_i: 0, dst_i: 0, src_ty: None, dst_ty: None, }, VariantCase { src_i: 1, dst_i: 1, src_ty, dst_ty, }, ] .into_iter(), ); } fn translate_result( &mut self, src_ty: TypeResultIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let src_ty = &self.types[src_ty]; let dst_ty = match dst_ty { InterfaceType::Result(t) => &self.types[*t], _ => panic!("expected a result"), }; let src_info = variant_info(self.types, [src_ty.ok.as_ref(), src_ty.err.as_ref()]); let dst_info = variant_info(self.types, [dst_ty.ok.as_ref(), dst_ty.err.as_ref()]); self.convert_variant( src, &src_info, dst, &dst_info, [ VariantCase { src_i: 0, dst_i: 0, src_ty: src_ty.ok.as_ref(), dst_ty: dst_ty.ok.as_ref(), }, VariantCase { src_i: 1, dst_i: 1, src_ty: src_ty.err.as_ref(), dst_ty: dst_ty.err.as_ref(), }, ] .into_iter(), ); } fn convert_variant<'a>( &mut self, src: &Source<'_>, src_info: &VariantInfo, dst: &Destination, dst_info: &VariantInfo, src_cases: impl ExactSizeIterator<Item = VariantCase<'a>>, ) { // The outermost block is special since it has the result type of the // translation here. That will depend on the `dst`. let outer_block_ty = match dst { Destination::Stack(dst_flat, _) => match dst_flat.len() { 0 => BlockType::Empty, 1 => BlockType::Result(dst_flat[0]), _ => { let ty = self.module.core_types.function(&[], &dst_flat); BlockType::FunctionType(ty) } }, Destination::Memory(_) => BlockType::Empty, }; self.instruction(Block(outer_block_ty)); // After the outermost block generate a new block for each of the // remaining cases. let src_cases_len = src_cases.len(); for _ in 0..src_cases_len - 1 { self.instruction(Block(BlockType::Empty)); } // Generate a block for an invalid variant discriminant self.instruction(Block(BlockType::Empty)); // And generate one final block that we'll be jumping out of with the // `br_table` self.instruction(Block(BlockType::Empty)); // Load the discriminant match src { Source::Stack(s) => self.stack_get(&s.slice(0..1), ValType::I32), Source::Memory(mem) => match src_info.size { DiscriminantSize::Size1 => self.i32_load8u(mem), DiscriminantSize::Size2 => self.i32_load16u(mem), DiscriminantSize::Size4 => self.i32_load(mem), }, } // Generate the `br_table` for the discriminant. Each case has an // offset of 1 to skip the trapping block. let mut targets = Vec::new(); for i in 0..src_cases_len { targets.push((i + 1) as u32); } self.instruction(BrTable(targets[..].into(), 0)); self.instruction(End); // end the `br_table` block self.trap(Trap::InvalidDiscriminant); self.instruction(End); // end the "invalid discriminant" block // Translate each case individually within its own block. Note that the // iteration order here places the first case in the innermost block // and the last case in the outermost block. This matches the order // of the jump targets in the `br_table` instruction. let src_cases_len = u32::try_from(src_cases_len).unwrap(); for case in src_cases { let VariantCase { src_i, src_ty, dst_i, dst_ty, } = case; // Translate the discriminant here, noting that `dst_i` may be // different than `src_i`. self.push_dst_addr(dst); self.instruction(I32Const(dst_i as i32)); match dst { Destination::Stack(stack, _) => self.stack_set(&stack[..1], ValType::I32), Destination::Memory(mem) => match dst_info.size { DiscriminantSize::Size1 => self.i32_store8(mem), DiscriminantSize::Size2 => self.i32_store16(mem), DiscriminantSize::Size4 => self.i32_store(mem), }, } let src_payload = src.payload_src(self.types, src_info, src_ty); let dst_payload = dst.payload_dst(self.types, dst_info, dst_ty); // Translate the payload of this case using the various types from // the dst/src. match (src_ty, dst_ty) { (Some(src_ty), Some(dst_ty)) => { self.translate(src_ty, &src_payload, dst_ty, &dst_payload); } (None, None) => {} _ => unimplemented!(), } // If the results of this translation were placed on the stack then // the stack values may need to be padded with more zeros due to // this particular case being possibly smaller than the entire // variant. That's handled here by pushing remaining zeros after // accounting for the discriminant pushed as well as the results of // this individual payload. if let Destination::Stack(payload_results, _) = dst_payload { if let Destination::Stack(dst_results, _) = dst { let remaining = &dst_results[1..][payload_results.len()..]; for ty in remaining { match ty { ValType::I32 => self.instruction(I32Const(0)), ValType::I64 => self.instruction(I64Const(0)), ValType::F32 => self.instruction(F32Const(0.0)), ValType::F64 => self.instruction(F64Const(0.0)), _ => unreachable!(), } } } } // Branch to the outermost block. Note that this isn't needed for // the outermost case since it simply falls through. if src_i != src_cases_len - 1 { self.instruction(Br(src_cases_len - src_i - 1)); } self.instruction(End); // end this case's block } } fn translate_own( &mut self, src_ty: TypeResourceTableIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let dst_ty = match dst_ty { InterfaceType::Own(t) => *t, _ => panic!("expected an `Own`"), }; let transfer = self.module.import_resource_transfer_own(); self.translate_resource(src_ty, src, dst_ty, dst, transfer); } fn translate_borrow( &mut self, src_ty: TypeResourceTableIndex, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination, ) { let dst_ty = match dst_ty { InterfaceType::Borrow(t) => *t, _ => panic!("expected an `Borrow`"), }; let transfer = self.module.import_resource_transfer_borrow(); self.translate_resource(src_ty, src, dst_ty, dst, transfer); } /// Translates the index `src`, which resides in the table `src_ty`, into /// and index within `dst_ty` and is stored at `dst`. /// /// Actual translation of the index happens in a wasmtime libcall, which a /// cranelift-generated trampoline to satisfy this import will call. The /// `transfer` function is an imported function which takes the src, src_ty, /// and dst_ty, and returns the dst index. fn translate_resource( &mut self, src_ty: TypeResourceTableIndex, src: &Source<'_>, dst_ty: TypeResourceTableIndex, dst: &Destination, transfer: FuncIndex, ) { self.push_dst_addr(dst); match src { Source::Memory(mem) => self.i32_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::I32), } self.instruction(I32Const(src_ty.as_u32() as i32)); self.instruction(I32Const(dst_ty.as_u32() as i32)); self.instruction(Call(transfer.as_u32())); match dst { Destination::Memory(mem) => self.i32_store(mem), Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32), } } fn trap_if_not_flag(&mut self, flags_global: GlobalIndex, flag_to_test: i32, trap: Trap) { self.instruction(GlobalGet(flags_global.as_u32())); self.instruction(I32Const(flag_to_test)); self.instruction(I32And); self.instruction(I32Eqz); self.instruction(If(BlockType::Empty)); self.trap(trap); self.instruction(End); } fn assert_not_flag(&mut self, flags_global: GlobalIndex, flag_to_test: i32, msg: &'static str) { self.instruction(GlobalGet(flags_global.as_u32())); self.instruction(I32Const(flag_to_test)); self.instruction(I32And); self.instruction(If(BlockType::Empty)); self.trap(Trap::AssertFailed(msg)); self.instruction(End); } fn set_flag(&mut self, flags_global: GlobalIndex, flag_to_set: i32, value: bool) { self.instruction(GlobalGet(flags_global.as_u32())); if value { self.instruction(I32Const(flag_to_set)); self.instruction(I32Or); } else { self.instruction(I32Const(!flag_to_set)); self.instruction(I32And); } self.instruction(GlobalSet(flags_global.as_u32())); } fn verify_aligned(&mut self, opts: &Options, addr_local: u32, align: u32) { // If the alignment is 1 then everything is trivially aligned and the // check can be omitted. if align == 1 { return; } self.instruction(LocalGet(addr_local)); assert!(align.is_power_of_two()); self.ptr_uconst(opts, align - 1); self.ptr_and(opts); self.ptr_if(opts, BlockType::Empty); self.trap(Trap::UnalignedPointer); self.instruction(End); } fn assert_aligned(&mut self, ty: &InterfaceType, mem: &Memory) { if !self.module.debug { return; } let align = self.types.align(mem.opts, ty); if align == 1 { return; } assert!(align.is_power_of_two()); self.instruction(LocalGet(mem.addr.idx)); self.ptr_uconst(mem.opts, mem.offset); self.ptr_add(mem.opts); self.ptr_uconst(mem.opts, align - 1); self.ptr_and(mem.opts); self.ptr_if(mem.opts, BlockType::Empty); self.trap(Trap::AssertFailed("pointer not aligned")); self.instruction(End); } fn malloc<'a>(&mut self, opts: &'a Options, size: MallocSize, align: u32) -> Memory<'a> { let realloc = opts.realloc.unwrap(); self.ptr_uconst(opts, 0); self.ptr_uconst(opts, 0); self.ptr_uconst(opts, align); match size { MallocSize::Const(size) => self.ptr_uconst(opts, size), MallocSize::Local(idx) => self.instruction(LocalGet(idx)), } self.instruction(Call(realloc.as_u32())); let addr = self.local_set_new_tmp(opts.ptr()); self.memory_operand(opts, addr, align) } fn memory_operand<'a>(&mut self, opts: &'a Options, addr: TempLocal, align: u32) -> Memory<'a> { let ret = Memory { addr, offset: 0, opts, }; self.verify_aligned(opts, ret.addr.idx, align); ret } /// Generates a new local in this function of the `ty` specified, /// initializing it with the top value on the current wasm stack. /// /// The returned `TempLocal` must be freed after it is finished with /// `free_temp_local`. fn local_tee_new_tmp(&mut self, ty: ValType) -> TempLocal { self.gen_temp_local(ty, LocalTee) } /// Same as `local_tee_new_tmp` but initializes the local with `LocalSet` /// instead of `LocalTee`. fn local_set_new_tmp(&mut self, ty: ValType) -> TempLocal { self.gen_temp_local(ty, LocalSet) } fn gen_temp_local(&mut self, ty: ValType, insn: fn(u32) -> Instruction<'static>) -> TempLocal { // First check to see if any locals are available in this function which // were previously generated but are no longer in use. if let Some(idx) = self.free_locals.get_mut(&ty).and_then(|v| v.pop()) { self.instruction(insn(idx)); return TempLocal { ty, idx, needs_free: true, }; } // Failing that generate a fresh new local. let locals = &mut self.module.funcs[self.result].locals; match locals.last_mut() { Some((cnt, prev_ty)) if ty == *prev_ty => *cnt += 1, _ => locals.push((1, ty)), } self.nlocals += 1; let idx = self.nlocals - 1; self.instruction(insn(idx)); TempLocal { ty, idx, needs_free: true, } } /// Used to release a `TempLocal` from a particular lexical scope to allow /// its possible reuse in later scopes. fn free_temp_local(&mut self, mut local: TempLocal) { assert!(local.needs_free); self.free_locals .entry(local.ty) .or_insert(Vec::new()) .push(local.idx); local.needs_free = false; } fn instruction(&mut self, instr: Instruction) { instr.encode(&mut self.code); } fn trap(&mut self, trap: Trap) { self.traps.push((self.code.len(), trap)); self.instruction(Unreachable); } /// Flushes out the current `code` instructions (and `traps` if there are /// any) into the destination function. /// /// This is a noop if no instructions have been encoded yet. fn flush_code(&mut self) { if self.code.is_empty() { return; } self.module.funcs[self.result].body.push(Body::Raw( mem::take(&mut self.code), mem::take(&mut self.traps), )); } fn finish(mut self) { // Append the final `end` instruction which all functions require, and // then empty out the temporary buffer in `Compiler`. self.instruction(End); self.flush_code(); // Flag the function as "done" which helps with an assert later on in // emission that everything was eventually finished. self.module.funcs[self.result].filled_in = true; } /// Fetches the value contained with the local specified by `stack` and /// converts it to `dst_ty`. /// /// This is only intended for use in primitive operations where `stack` is /// guaranteed to have only one local. The type of the local on the stack is /// then converted to `dst_ty` appropriately. Note that the types may be /// different due to the "flattening" of variant types. fn stack_get(&mut self, stack: &Stack<'_>, dst_ty: ValType) { assert_eq!(stack.locals.len(), 1); let (idx, src_ty) = stack.locals[0]; self.instruction(LocalGet(idx)); match (src_ty, dst_ty) { (ValType::I32, ValType::I32) | (ValType::I64, ValType::I64) | (ValType::F32, ValType::F32) | (ValType::F64, ValType::F64) => {} (ValType::I32, ValType::F32) => self.instruction(F32ReinterpretI32), (ValType::I64, ValType::I32) => { self.assert_i64_upper_bits_not_set(idx); self.instruction(I32WrapI64); } (ValType::I64, ValType::F64) => self.instruction(F64ReinterpretI64), (ValType::I64, ValType::F32) => { self.assert_i64_upper_bits_not_set(idx); self.instruction(I32WrapI64); self.instruction(F32ReinterpretI32); } // should not be possible given the `join` function for variants (ValType::I32, ValType::I64) | (ValType::I32, ValType::F64) | (ValType::F32, ValType::I32) | (ValType::F32, ValType::I64) | (ValType::F32, ValType::F64) | (ValType::F64, ValType::I32) | (ValType::F64, ValType::I64) | (ValType::F64, ValType::F32) // not used in the component model | (ValType::Ref(_), _) | (_, ValType::Ref(_)) | (ValType::V128, _) | (_, ValType::V128) => { panic!("cannot get {dst_ty:?} from {src_ty:?} local"); } } } fn assert_i64_upper_bits_not_set(&mut self, local: u32) { if !self.module.debug { return; } self.instruction(LocalGet(local)); self.instruction(I64Const(32)); self.instruction(I64ShrU); self.instruction(I32WrapI64); self.instruction(If(BlockType::Empty)); self.trap(Trap::AssertFailed("upper bits are unexpectedly set")); self.instruction(End); } /// Converts the top value on the WebAssembly stack which has type /// `src_ty` to `dst_tys[0]`. /// /// This is only intended for conversion of primitives where the `dst_tys` /// list is known to be of length 1. fn stack_set(&mut self, dst_tys: &[ValType], src_ty: ValType) { assert_eq!(dst_tys.len(), 1); let dst_ty = dst_tys[0]; match (src_ty, dst_ty) { (ValType::I32, ValType::I32) | (ValType::I64, ValType::I64) | (ValType::F32, ValType::F32) | (ValType::F64, ValType::F64) => {} (ValType::F32, ValType::I32) => self.instruction(I32ReinterpretF32), (ValType::I32, ValType::I64) => self.instruction(I64ExtendI32U), (ValType::F64, ValType::I64) => self.instruction(I64ReinterpretF64), (ValType::F32, ValType::I64) => { self.instruction(I32ReinterpretF32); self.instruction(I64ExtendI32U); } // should not be possible given the `join` function for variants (ValType::I64, ValType::I32) | (ValType::F64, ValType::I32) | (ValType::I32, ValType::F32) | (ValType::I64, ValType::F32) | (ValType::F64, ValType::F32) | (ValType::I32, ValType::F64) | (ValType::I64, ValType::F64) | (ValType::F32, ValType::F64) // not used in the component model | (ValType::Ref(_), _) | (_, ValType::Ref(_)) | (ValType::V128, _) | (_, ValType::V128) => { panic!("cannot get {dst_ty:?} from {src_ty:?} local"); } } } fn i32_load8u(&mut self, mem: &Memory) { self.instruction(LocalGet(mem.addr.idx)); self.instruction(I32Load8U(mem.memarg(0))); } fn i32_load8s(&mut self, mem: &Memory) { self.instruction(LocalGet(mem.addr.idx)); self.instruction(I32Load8S(mem.memarg(0))); } fn i32_load16u(&mut self, mem: &Memory) { self.instruction(LocalGet(mem.addr.idx)); self.instruction(I32Load16U(mem.memarg(1))); } fn i32_load16s(&mut self, mem: &Memory) { self.instruction(LocalGet(mem.addr.idx)); self.instruction(I32Load16S(mem.memarg(1))); } fn i32_load(&mut self, mem: &Memory) { self.instruction(LocalGet(mem.addr.idx)); self.instruction(I32Load(mem.memarg(2))); } fn i64_load(&mut self, mem: &Memory) { self.instruction(LocalGet(mem.addr.idx)); self.instruction(I64Load(mem.memarg(3))); } fn ptr_load(&mut self, mem: &Memory) { if mem.opts.memory64 { self.i64_load(mem); } else { self.i32_load(mem); } } fn ptr_add(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Add); } else { self.instruction(I32Add); } } fn ptr_sub(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Sub); } else { self.instruction(I32Sub); } } fn ptr_mul(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Mul); } else { self.instruction(I32Mul); } } fn ptr_ge_u(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64GeU); } else { self.instruction(I32GeU); } } fn ptr_lt_u(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64LtU); } else { self.instruction(I32LtU); } } fn ptr_shl(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Shl); } else { self.instruction(I32Shl); } } fn ptr_eqz(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Eqz); } else { self.instruction(I32Eqz); } } fn ptr_uconst(&mut self, opts: &Options, val: u32) { if opts.memory64 { self.instruction(I64Const(val.into())); } else { self.instruction(I32Const(val as i32)); } } fn ptr_iconst(&mut self, opts: &Options, val: i32) { if opts.memory64 { self.instruction(I64Const(val.into())); } else { self.instruction(I32Const(val)); } } fn ptr_eq(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Eq); } else { self.instruction(I32Eq); } } fn ptr_ne(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Ne); } else { self.instruction(I32Ne); } } fn ptr_and(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64And); } else { self.instruction(I32And); } } fn ptr_or(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Or); } else { self.instruction(I32Or); } } fn ptr_xor(&mut self, opts: &Options) { if opts.memory64 { self.instruction(I64Xor); } else { self.instruction(I32Xor); } } fn ptr_if(&mut self, opts: &Options, ty: BlockType) { if opts.memory64 { self.instruction(I64Const(0)); self.instruction(I64Ne); } self.instruction(If(ty)); } fn ptr_br_if(&mut self, opts: &Options, depth: u32) { if opts.memory64 { self.instruction(I64Const(0)); self.instruction(I64Ne); } self.instruction(BrIf(depth)); } fn f32_load(&mut self, mem: &Memory) { self.instruction(LocalGet(mem.addr.idx)); self.instruction(F32Load(mem.memarg(2))); } fn f64_load(&mut self, mem: &Memory) { self.instruction(LocalGet(mem.addr.idx)); self.instruction(F64Load(mem.memarg(3))); } fn push_dst_addr(&mut self, dst: &Destination) { if let Destination::Memory(mem) = dst { self.instruction(LocalGet(mem.addr.idx)); } } fn i32_store8(&mut self, mem: &Memory) { self.instruction(I32Store8(mem.memarg(0))); } fn i32_store16(&mut self, mem: &Memory) { self.instruction(I32Store16(mem.memarg(1))); } fn i32_store(&mut self, mem: &Memory) { self.instruction(I32Store(mem.memarg(2))); } fn i64_store(&mut self, mem: &Memory) { self.instruction(I64Store(mem.memarg(3))); } fn ptr_store(&mut self, mem: &Memory) { if mem.opts.memory64 { self.i64_store(mem); } else { self.i32_store(mem); } } fn f32_store(&mut self, mem: &Memory) { self.instruction(F32Store(mem.memarg(2))); } fn f64_store(&mut self, mem: &Memory) { self.instruction(F64Store(mem.memarg(3))); } } impl<'a> Source<'a> { /// Given this `Source` returns an iterator over the `Source` for each of /// the component `fields` specified. /// /// This will automatically slice stack-based locals to the appropriate /// width for each component type and additionally calculate the appropriate /// offset for each memory-based type. fn record_field_srcs<'b>( &'b self, types: &'b ComponentTypesBuilder, fields: impl IntoIterator<Item = InterfaceType> + 'b, ) -> impl Iterator<Item = Source<'a>> + 'b where 'a: 'b, { let mut offset = 0; fields.into_iter().map(move |ty| match self { Source::Memory(mem) => { let mem = next_field_offset(&mut offset, types, &ty, mem); Source::Memory(mem) } Source::Stack(stack) => { let cnt = types.flat_types(&ty).unwrap().len() as u32; offset += cnt; Source::Stack(stack.slice((offset - cnt) as usize..offset as usize)) } }) } /// Returns the corresponding discriminant source and payload source f fn payload_src( &self, types: &ComponentTypesBuilder, info: &VariantInfo, case: Option<&InterfaceType>, ) -> Source<'a> { match self { Source::Stack(s) => { let flat_len = match case { Some(case) => types.flat_types(case).unwrap().len(), None => 0, }; Source::Stack(s.slice(1..s.locals.len()).slice(0..flat_len)) } Source::Memory(mem) => { let mem = if mem.opts.memory64 { mem.bump(info.payload_offset64) } else { mem.bump(info.payload_offset32) }; Source::Memory(mem) } } } fn opts(&self) -> &'a Options { match self { Source::Stack(s) => s.opts, Source::Memory(mem) => mem.opts, } } } impl<'a> Destination<'a> { /// Same as `Source::record_field_srcs` but for destinations. fn record_field_dsts<'b>( &'b self, types: &'b ComponentTypesBuilder, fields: impl IntoIterator<Item = InterfaceType> + 'b, ) -> impl Iterator<Item = Destination<'b>> + 'b where 'a: 'b, { let mut offset = 0; fields.into_iter().map(move |ty| match self { Destination::Memory(mem) => { let mem = next_field_offset(&mut offset, types, &ty, mem); Destination::Memory(mem) } Destination::Stack(s, opts) => { let cnt = types.flat_types(&ty).unwrap().len() as u32; offset += cnt; Destination::Stack(&s[(offset - cnt) as usize..offset as usize], opts) } }) } /// Returns the corresponding discriminant source and payload source f fn payload_dst( &self, types: &ComponentTypesBuilder, info: &VariantInfo, case: Option<&InterfaceType>, ) -> Destination { match self { Destination::Stack(s, opts) => { let flat_len = match case { Some(case) => types.flat_types(case).unwrap().len(), None => 0, }; Destination::Stack(&s[1..][..flat_len], opts) } Destination::Memory(mem) => { let mem = if mem.opts.memory64 { mem.bump(info.payload_offset64) } else { mem.bump(info.payload_offset32) }; Destination::Memory(mem) } } } fn opts(&self) -> &'a Options { match self { Destination::Stack(_, opts) => opts, Destination::Memory(mem) => mem.opts, } } } fn next_field_offset<'a>( offset: &mut u32, types: &ComponentTypesBuilder, field: &InterfaceType, mem: &Memory<'a>, ) -> Memory<'a> { let abi = types.canonical_abi(field); let offset = if mem.opts.memory64 { abi.next_field64(offset) } else { abi.next_field32(offset) }; mem.bump(offset) } impl<'a> Memory<'a> { fn memarg(&self, align: u32) -> MemArg { MemArg { offset: u64::from(self.offset), align, memory_index: self.opts.memory.unwrap().as_u32(), } } fn bump(&self, offset: u32) -> Memory<'a> { Memory { opts: self.opts, addr: TempLocal::new(self.addr.idx, self.addr.ty), offset: self.offset + offset, } } } impl<'a> Stack<'a> { fn slice(&self, range: Range<usize>) -> Stack<'a> { Stack { locals: &self.locals[range], opts: self.opts, } } } struct VariantCase<'a> { src_i: u32, src_ty: Option<&'a InterfaceType>, dst_i: u32, dst_ty: Option<&'a InterfaceType>, } fn variant_info<'a, I>(types: &ComponentTypesBuilder, cases: I) -> VariantInfo where I: IntoIterator<Item = Option<&'a InterfaceType>>, I::IntoIter: ExactSizeIterator, { VariantInfo::new( cases .into_iter() .map(|ty| ty.map(|ty| types.canonical_abi(ty))), ) .0 } enum MallocSize { Const(u32), Local(u32), } struct WasmString<'a> { ptr: TempLocal, len: TempLocal, opts: &'a Options, } struct TempLocal { idx: u32, ty: ValType, needs_free: bool, } impl TempLocal { fn new(idx: u32, ty: ValType) -> TempLocal { TempLocal { idx, ty, needs_free: false, } } } impl std::ops::Drop for TempLocal { fn drop(&mut self) { if self.needs_free { panic!("temporary local not free'd"); } } } impl From<FlatType> for ValType { fn from(ty: FlatType) -> ValType { match ty { FlatType::I32 => ValType::I32, FlatType::I64 => ValType::I64, FlatType::F32 => ValType::F32, FlatType::F64 => ValType::F64, } } }