diff options
Diffstat (limited to 'yjit')
-rw-r--r-- | yjit/src/backend/arm64/mod.rs | 426 | ||||
-rw-r--r-- | yjit/src/backend/ir.rs | 888 | ||||
-rw-r--r-- | yjit/src/backend/tests.rs | 8 | ||||
-rw-r--r-- | yjit/src/backend/x86_64/mod.rs | 431 |
4 files changed, 960 insertions, 793 deletions
diff --git a/yjit/src/backend/arm64/mod.rs b/yjit/src/backend/arm64/mod.rs index a32be6a6b2..60cdf2b9d1 100644 --- a/yjit/src/backend/arm64/mod.rs +++ b/yjit/src/backend/arm64/mod.rs @@ -59,6 +59,13 @@ impl From<Opnd> for A64Opnd { } } +/// Also implement going from a reference to an operand for convenience. +impl From<&Opnd> for A64Opnd { + fn from(opnd: &Opnd) -> Self { + A64Opnd::from(*opnd) + } +} + impl Assembler { // A special scratch register for intermediate processing. @@ -182,6 +189,41 @@ impl Assembler } } + /// Returns the operands that should be used for a boolean logic + /// instruction. + fn split_boolean_operands(asm: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> (Opnd, Opnd) { + match (opnd0, opnd1) { + (Opnd::Reg(_), Opnd::Reg(_)) => { + (opnd0, opnd1) + }, + (reg_opnd @ Opnd::Reg(_), other_opnd) | + (other_opnd, reg_opnd @ Opnd::Reg(_)) => { + let opnd1 = split_bitmask_immediate(asm, other_opnd); + (reg_opnd, opnd1) + }, + _ => { + let opnd0 = split_load_operand(asm, opnd0); + let opnd1 = split_bitmask_immediate(asm, opnd1); + (opnd0, opnd1) + } + } + } + + /// Returns the operands that should be used for a csel instruction. + fn split_csel_operands(asm: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> (Opnd, Opnd) { + let opnd0 = match opnd0 { + Opnd::Reg(_) | Opnd::InsnOut { .. } => opnd0, + _ => split_load_operand(asm, opnd0) + }; + + let opnd1 = match opnd1 { + Opnd::Reg(_) | Opnd::InsnOut { .. } => opnd1, + _ => split_load_operand(asm, opnd1) + }; + + (opnd0, opnd1) + } + let mut asm_local = Assembler::new_with_label_names(std::mem::take(&mut self.label_names)); let asm = &mut asm_local; let mut iterator = self.into_draining_iter(); @@ -192,7 +234,7 @@ impl Assembler // such that only the Op::Load instruction needs to handle that // case. If the values aren't heap objects then we'll treat them as // if they were just unsigned integer. - let skip_load = matches!(insn, Insn { op: Op::Load, .. }); + let skip_load = matches!(insn, Insn::Load { .. }); let mut opnd_iter = insn.opnd_iter_mut(); while let Some(opnd) = opnd_iter.next() { @@ -209,10 +251,10 @@ impl Assembler } match insn { - Insn { op: Op::Add, opnds, .. } => { - match (opnds[0], opnds[1]) { + Insn::Add { left, right, .. } => { + match (left, right) { (Opnd::Reg(_) | Opnd::InsnOut { .. }, Opnd::Reg(_) | Opnd::InsnOut { .. }) => { - asm.add(opnds[0], opnds[1]); + asm.add(left, right); }, (reg_opnd @ (Opnd::Reg(_) | Opnd::InsnOut { .. }), other_opnd) | (other_opnd, reg_opnd @ (Opnd::Reg(_) | Opnd::InsnOut { .. })) => { @@ -220,30 +262,25 @@ impl Assembler asm.add(reg_opnd, opnd1); }, _ => { - let opnd0 = split_load_operand(asm, opnds[0]); - let opnd1 = split_shifted_immediate(asm, opnds[1]); + let opnd0 = split_load_operand(asm, left); + let opnd1 = split_shifted_immediate(asm, right); asm.add(opnd0, opnd1); } } }, - Insn { op: Op::And | Op::Or | Op::Xor, opnds, target, text, pos_marker, .. } => { - match (opnds[0], opnds[1]) { - (Opnd::Reg(_), Opnd::Reg(_)) => { - asm.push_insn_parts(insn.op, vec![opnds[0], opnds[1]], target, text, pos_marker); - }, - (reg_opnd @ Opnd::Reg(_), other_opnd) | - (other_opnd, reg_opnd @ Opnd::Reg(_)) => { - let opnd1 = split_bitmask_immediate(asm, other_opnd); - asm.push_insn_parts(insn.op, vec![reg_opnd, opnd1], target, text, pos_marker); - }, - _ => { - let opnd0 = split_load_operand(asm, opnds[0]); - let opnd1 = split_bitmask_immediate(asm, opnds[1]); - asm.push_insn_parts(insn.op, vec![opnd0, opnd1], target, text, pos_marker); - } - } + Insn::And { left, right, .. } => { + let (opnd0, opnd1) = split_boolean_operands(asm, left, right); + asm.and(opnd0, opnd1); + }, + Insn::Or { left, right, .. } => { + let (opnd0, opnd1) = split_boolean_operands(asm, left, right); + asm.or(opnd0, opnd1); }, - Insn { op: Op::CCall, opnds, target, .. } => { + Insn::Xor { left, right, .. } => { + let (opnd0, opnd1) = split_boolean_operands(asm, left, right); + asm.xor(opnd0, opnd1); + }, + Insn::CCall { opnds, target, .. } => { assert!(opnds.len() <= C_ARG_OPNDS.len()); // For each of the operands we're going to first load them @@ -258,60 +295,82 @@ impl Assembler // Now we push the CCall without any arguments so that it // just performs the call. - asm.ccall(target.unwrap().unwrap_fun_ptr(), vec![]); + asm.ccall(target.unwrap_fun_ptr(), vec![]); }, - Insn { op: Op::Cmp, opnds, .. } => { - let opnd0 = match opnds[0] { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnds[0], - _ => split_load_operand(asm, opnds[0]) + Insn::Cmp { left, right } => { + let opnd0 = match left { + Opnd::Reg(_) | Opnd::InsnOut { .. } => left, + _ => split_load_operand(asm, left) }; - let opnd1 = split_shifted_immediate(asm, opnds[1]); + let opnd1 = split_shifted_immediate(asm, right); asm.cmp(opnd0, opnd1); }, - Insn { op: Op::CRet, opnds, .. } => { - if opnds[0] != Opnd::Reg(C_RET_REG) { - let value = split_load_operand(asm, opnds[0]); + Insn::CRet(opnd) => { + if opnd != Opnd::Reg(C_RET_REG) { + let value = split_load_operand(asm, opnd); asm.mov(C_RET_OPND, value); } asm.cret(C_RET_OPND); }, - Insn { op: Op::CSelZ | Op::CSelNZ | Op::CSelE | Op::CSelNE | Op::CSelL | Op::CSelLE | Op::CSelG | Op::CSelGE, opnds, target, text, pos_marker, .. } => { - let new_opnds = opnds.into_iter().map(|opnd| { - match opnd { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnd, - _ => split_load_operand(asm, opnd) - } - }).collect(); - - asm.push_insn_parts(insn.op, new_opnds, target, text, pos_marker); + Insn::CSelZ { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_z(opnd0, opnd1); }, - Insn { op: Op::IncrCounter, opnds, .. } => { + Insn::CSelNZ { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_nz(opnd0, opnd1); + }, + Insn::CSelE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_e(opnd0, opnd1); + }, + Insn::CSelNE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_ne(opnd0, opnd1); + }, + Insn::CSelL { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_l(opnd0, opnd1); + }, + Insn::CSelLE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_le(opnd0, opnd1); + }, + Insn::CSelG { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_g(opnd0, opnd1); + }, + Insn::CSelGE { truthy, falsy, .. } => { + let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); + asm.csel_ge(opnd0, opnd1); + }, + Insn::IncrCounter { mem, value } => { // We'll use LDADD later which only works with registers // ... Load pointer into register - let counter_addr = split_lea_operand(asm, opnds[0]); + let counter_addr = split_lea_operand(asm, mem); // Load immediates into a register - let addend = match opnds[1] { + let addend = match value { opnd @ Opnd::Imm(_) | opnd @ Opnd::UImm(_) => asm.load(opnd), opnd => opnd, }; asm.incr_counter(counter_addr, addend); }, - Insn { op: Op::JmpOpnd, opnds, .. } => { - if let Opnd::Mem(_) = opnds[0] { - let opnd0 = split_load_operand(asm, opnds[0]); + Insn::JmpOpnd(opnd) => { + if let Opnd::Mem(_) = opnd { + let opnd0 = split_load_operand(asm, opnd); asm.jmp_opnd(opnd0); } else { - asm.jmp_opnd(opnds[0]); + asm.jmp_opnd(opnd); } }, - Insn { op: Op::Load, opnds, .. } => { - split_load_operand(asm, opnds[0]); + Insn::Load { opnd, .. } => { + split_load_operand(asm, opnd); }, - Insn { op: Op::LoadSExt, opnds, .. } => { - match opnds[0] { + Insn::LoadSExt { opnd, .. } => { + match opnd { // We only want to sign extend if the operand is a // register, instruction output, or memory address that // is 32 bits. Otherwise we'll just load the value @@ -319,87 +378,87 @@ impl Assembler Opnd::Reg(Reg { num_bits: 32, .. }) | Opnd::InsnOut { num_bits: 32, .. } | Opnd::Mem(Mem { num_bits: 32, .. }) => { - asm.load_sext(opnds[0]); + asm.load_sext(opnd); }, _ => { - asm.load(opnds[0]); + asm.load(opnd); } }; }, - Insn { op: Op::Mov, opnds, .. } => { - let value = match (opnds[0], opnds[1]) { + Insn::Mov { dest, src } => { + let value = match (dest, src) { // If the first operand is a memory operand, we're going // to transform this into a store instruction, so we'll // need to load this anyway. - (Opnd::Mem(_), Opnd::UImm(_)) => asm.load(opnds[1]), + (Opnd::Mem(_), Opnd::UImm(_)) => asm.load(src), // The value that is being moved must be either a // register or an immediate that can be encoded as a // bitmask immediate. Otherwise, we'll need to split the // move into multiple instructions. - _ => split_bitmask_immediate(asm, opnds[1]) + _ => split_bitmask_immediate(asm, src) }; // If we're attempting to load into a memory operand, then // we'll switch over to the store instruction. Otherwise // we'll use the normal mov instruction. - match opnds[0] { + match dest { Opnd::Mem(_) => { - let opnd0 = split_memory_address(asm, opnds[0]); + let opnd0 = split_memory_address(asm, dest); asm.store(opnd0, value); }, Opnd::Reg(_) => { - asm.mov(opnds[0], value); + asm.mov(dest, value); }, _ => unreachable!() }; }, - Insn { op: Op::Not, opnds, .. } => { + Insn::Not { opnd, .. } => { // The value that is being negated must be in a register, so // if we get anything else we need to load it first. - let opnd0 = match opnds[0] { - Opnd::Mem(_) => split_load_operand(asm, opnds[0]), - _ => opnds[0] + let opnd0 = match opnd { + Opnd::Mem(_) => split_load_operand(asm, opnd), + _ => opnd }; asm.not(opnd0); }, - Insn { op: Op::Store, opnds, .. } => { + Insn::Store { dest, src } => { // The displacement for the STUR instruction can't be more // than 9 bits long. If it's longer, we need to load the // memory address into a register first. - let opnd0 = split_memory_address(asm, opnds[0]); + let opnd0 = split_memory_address(asm, dest); // The value being stored must be in a register, so if it's // not already one we'll load it first. - let opnd1 = match opnds[1] { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnds[1], - _ => split_load_operand(asm, opnds[1]) + let opnd1 = match src { + Opnd::Reg(_) | Opnd::InsnOut { .. } => src, + _ => split_load_operand(asm, src) }; asm.store(opnd0, opnd1); }, - Insn { op: Op::Sub, opnds, .. } => { - let opnd0 = match opnds[0] { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnds[0], - _ => split_load_operand(asm, opnds[0]) + Insn::Sub { left, right, .. } => { + let opnd0 = match left { + Opnd::Reg(_) | Opnd::InsnOut { .. } => left, + _ => split_load_operand(asm, left) }; - let opnd1 = split_shifted_immediate(asm, opnds[1]); + let opnd1 = split_shifted_immediate(asm, right); asm.sub(opnd0, opnd1); }, - Insn { op: Op::Test, opnds, .. } => { + Insn::Test { left, right } => { // The value being tested must be in a register, so if it's // not already one we'll load it first. - let opnd0 = match opnds[0] { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnds[0], - _ => split_load_operand(asm, opnds[0]) + let opnd0 = match left { + Opnd::Reg(_) | Opnd::InsnOut { .. } => left, + _ => split_load_operand(asm, left) }; // The second value must be either a register or an // unsigned immediate that can be encoded as a bitmask // immediate. If it's not one of those, we'll need to load // it first. - let opnd1 = split_bitmask_immediate(asm, opnds[1]); + let opnd1 = split_bitmask_immediate(asm, right); asm.test(opnd0, opnd1); }, _ => { @@ -589,23 +648,20 @@ impl Assembler let start_write_pos = cb.get_write_pos(); for insn in &self.insns { match insn { - Insn { op: Op::Comment, text, .. } => { + Insn::Comment(text) => { if cfg!(feature = "asm_comments") { - cb.add_comment(text.as_ref().unwrap()); + cb.add_comment(text); } }, - Insn { op: Op::Label, target, .. } => { - cb.write_label(target.unwrap().unwrap_label_idx()); + Insn::Label(target) => { + cb.write_label(target.unwrap_label_idx()); }, // Report back the current position in the generated code - Insn { op: Op::PosMarker, pos_marker, .. } => { - let pos = cb.get_write_ptr(); - let pos_marker_fn = pos_marker.as_ref().unwrap(); - pos_marker_fn(pos); + Insn::PosMarker(pos_marker) => { + pos_marker(cb.get_write_ptr()); } - Insn { op: Op::BakeString, text, .. } => { - let str = text.as_ref().unwrap(); - for byte in str.as_bytes() { + Insn::BakeString(text) => { + for byte in text.as_bytes() { cb.write_byte(*byte); } @@ -615,69 +671,69 @@ impl Assembler // Pad out the string to the next 4-byte boundary so that // it's easy to jump past. - for _ in 0..(4 - ((str.len() + 1) % 4)) { + for _ in 0..(4 - ((text.len() + 1) % 4)) { cb.write_byte(0); } }, - Insn { op: Op::Add, opnds, out, .. } => { - adds(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::Add { left, right, out } => { + adds(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::FrameSetup, .. } => { + Insn::FrameSetup => { stp_pre(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, -16)); // X29 (frame_pointer) = SP mov(cb, X29, C_SP_REG); }, - Insn { op: Op::FrameTeardown, .. } => { + Insn::FrameTeardown => { // SP = X29 (frame pointer) mov(cb, C_SP_REG, X29); ldp_post(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, 16)); }, - Insn { op: Op::Sub, opnds, out, .. } => { - subs(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::Sub { left, right, out } => { + subs(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::And, opnds, out, .. } => { - and(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::And { left, right, out } => { + and(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::Or, opnds, out, .. } => { - orr(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::Or { left, right, out } => { + orr(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::Xor, opnds, out, .. } => { - eor(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::Xor { left, right, out } => { + eor(cb, out.into(), left.into(), right.into()); }, - Insn { op: Op::Not, opnds, out, .. } => { - mvn(cb, (*out).into(), opnds[0].into()); + Insn::Not { opnd, out } => { + mvn(cb, out.into(), opnd.into()); }, - Insn { op: Op::RShift, opnds, out, .. } => { - asr(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::RShift { opnd, shift, out } => { + asr(cb, out.into(), opnd.into(), shift.into()); }, - Insn { op: Op::URShift, opnds, out, .. } => { - lsr(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::URShift { opnd, shift, out } => { + lsr(cb, out.into(), opnd.into(), shift.into()); }, - Insn { op: Op::LShift, opnds, out, .. } => { - lsl(cb, (*out).into(), opnds[0].into(), opnds[1].into()); + Insn::LShift { opnd, shift, out } => { + lsl(cb, out.into(), opnd.into(), shift.into()); }, - Insn { op: Op::Store, opnds, .. } => { + Insn::Store { dest, src } => { // This order may be surprising but it is correct. The way // the Arm64 assembler works, the register that is going to // be stored is first and the address is second. However in // our IR we have the address first and the register second. - stur(cb, opnds[1].into(), opnds[0].into()); + stur(cb, src.into(), dest.into()); }, - Insn { op: Op::Load, opnds, out, .. } => { - match opnds[0] { + Insn::Load { opnd, out } => { + match *opnd { Opnd::Reg(_) | Opnd::InsnOut { .. } => { - mov(cb, (*out).into(), opnds[0].into()); + mov(cb, out.into(), opnd.into()); }, Opnd::UImm(uimm) => { - emit_load_value(cb, (*out).into(), uimm); + emit_load_value(cb, out.into(), uimm); }, Opnd::Imm(imm) => { - emit_load_value(cb, (*out).into(), imm as u64); + emit_load_value(cb, out.into(), imm as u64); }, Opnd::Mem(_) => { - ldur(cb, (*out).into(), opnds[0].into()); + ldur(cb, out.into(), opnd.into()); }, Opnd::Value(value) => { // We dont need to check if it's a special const @@ -689,7 +745,7 @@ impl Assembler // references to GC'd Value operands. If the value // being loaded is a heap object, we'll report that // back out to the gc_offsets list. - ldr_literal(cb, (*out).into(), 2); + ldr_literal(cb, out.into(), 2); b(cb, A64Opnd::new_imm(1 + (SIZEOF_VALUE as i64) / 4)); cb.write_bytes(&value.as_u64().to_le_bytes()); @@ -701,29 +757,29 @@ impl Assembler } }; }, - Insn { op: Op::LoadSExt, opnds, out, .. } => { - match opnds[0] { + Insn::LoadSExt { opnd, out } => { + match *opnd { Opnd::Reg(Reg { num_bits: 32, .. }) | Opnd::InsnOut { num_bits: 32, .. } => { - sxtw(cb, (*out).into(), opnds[0].into()); + sxtw(cb, out.into(), opnd.into()); }, Opnd::Mem(Mem { num_bits: 32, .. }) => { - ldursw(cb, (*out).into(), opnds[0].into()); + ldursw(cb, out.into(), opnd.into()); }, _ => unreachable!() }; }, - Insn { op: Op::Mov, opnds, .. } => { - mov(cb, opnds[0].into(), opnds[1].into()); + Insn::Mov { dest, src } => { + mov(cb, dest.into(), src.into()); }, - Insn { op: Op::Lea, opnds, out, .. } => { - let opnd: A64Opnd = opnds[0].into(); + Insn::Lea { opnd, out } => { + let opnd: A64Opnd = opnd.into(); match opnd { A64Opnd::Mem(mem) => { add( cb, - (*out).into(), + out.into(), A64Opnd::Reg(A64Reg { reg_no: mem.base_reg_no, num_bits: 64 }), A64Opnd::new_imm(mem.disp.into()) ); @@ -733,25 +789,25 @@ impl Assembler } }; }, - Insn { op: Op::LeaLabel, out, target, .. } => { - let label_idx = target.unwrap().unwrap_label_idx(); + Insn::LeaLabel { out, target, .. } => { + let label_idx = target.unwrap_label_idx(); cb.label_ref(label_idx, 4, |cb, end_addr, dst_addr| { adr(cb, Self::SCRATCH0, A64Opnd::new_imm(dst_addr - (end_addr - 4))); }); - mov(cb, (*out).into(), Self::SCRATCH0); + mov(cb, out.into(), Self::SCRATCH0); }, - Insn { op: Op::CPush, opnds, .. } => { - emit_push(cb, opnds[0].into()); + Insn::CPush(opnd) => { + emit_push(cb, opnd.into()); }, - Insn { op: Op::CPop, out, .. } => { - emit_pop(cb, (*out).into()); + Insn::CPop { out } => { + emit_pop(cb, out.into()); }, - Insn { op: Op::CPopInto, opnds, .. } => { - emit_pop(cb, opnds[0].into()); + Insn::CPopInto(opnd) => { + emit_pop(cb, opnd.into()); }, - Insn { op: Op::CPushAll, .. } => { + Insn::CPushAll => { let regs = Assembler::get_caller_save_regs(); for reg in regs { @@ -762,7 +818,7 @@ impl Assembler mrs(cb, Self::SCRATCH0, SystemRegister::NZCV); emit_push(cb, Self::SCRATCH0); }, - Insn { op: Op::CPopAll, .. } => { + Insn::CPopAll => { let regs = Assembler::get_caller_save_regs(); // Pop the state/flags register @@ -773,10 +829,10 @@ impl Assembler emit_pop(cb, A64Opnd::Reg(reg)); } }, - Insn { op: Op::CCall, target, .. } => { + Insn::CCall { target, .. } => { // The offset to the call target in bytes let src_addr = cb.get_write_ptr().into_i64(); - let dst_addr = target.unwrap().unwrap_fun_ptr() as i64; + let dst_addr = target.unwrap_fun_ptr() as i64; let offset = dst_addr - src_addr; // The offset in instruction count for BL's immediate let offset = offset / 4; @@ -790,20 +846,20 @@ impl Assembler blr(cb, Self::SCRATCH0); } }, - Insn { op: Op::CRet, .. } => { + Insn::CRet { .. } => { ret(cb, A64Opnd::None); }, - Insn { op: Op::Cmp, opnds, .. } => { - cmp(cb, opnds[0].into(), opnds[1].into()); + Insn::Cmp { left, right } => { + cmp(cb, left.into(), right.into()); }, - Insn { op: Op::Test, opnds, .. } => { - tst(cb, opnds[0].into(), opnds[1].into()); + Insn::Test { left, right } => { + tst(cb, left.into(), right.into()); }, - Insn { op: Op::JmpOpnd, opnds, .. } => { - br(cb, opnds[0].into()); + Insn::JmpOpnd(opnd) => { + br(cb, opnd.into()); }, - Insn { op: Op::Jmp, target, .. } => { - match target.unwrap() { + Insn::Jmp(target) => { + match target { Target::CodePtr(dst_ptr) => { let src_addr = cb.get_write_ptr().into_i64(); let dst_addr = dst_ptr.into_i64(); @@ -831,60 +887,62 @@ impl Assembler // instruction once we know the offset. We're going // to assume we can fit into a single b instruction. // It will panic otherwise. - cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| { + cb.label_ref(*label_idx, 4, |cb, src_addr, dst_addr| { b(cb, A64Opnd::new_imm((dst_addr - (src_addr - 4)) / 4)); }); }, _ => unreachable!() }; }, - Insn { op: Op::Je, target, .. } => { - emit_conditional_jump::<{Condition::EQ}>(cb, target.unwrap()); + Insn::Je(target) => { + emit_conditional_jump::<{Condition::EQ}>(cb, *target); }, - Insn { op: Op::Jne, target, .. } => { - emit_conditional_jump::<{Condition::NE}>(cb, target.unwrap()); + Insn::Jne(target) => { + emit_conditional_jump::<{Condition::NE}>(cb, *target); }, - Insn { op: Op::Jl, target, .. } => { - emit_conditional_jump::<{Condition::LT}>(cb, target.unwrap()); + Insn::Jl(target) => { + emit_conditional_jump::<{Condition::LT}>(cb, *target); }, - Insn { op: Op::Jbe, target, .. } => { - emit_conditional_jump::<{Condition::LS}>(cb, target.unwrap()); + Insn::Jbe(target) => { + emit_conditional_jump::<{Condition::LS}>(cb, *target); }, - Insn { op: Op::Jz, target, .. } => { - emit_conditional_jump::<{Condition::EQ}>(cb, target.unwrap()); + Insn::Jz(target) => { + emit_conditional_jump::<{Condition::EQ}>(cb, *target); }, - Insn { op: Op::Jnz, target, .. } => { - emit_conditional_jump::<{Condition::NE}>(cb, target.unwrap()); + Insn::Jnz(target) => { + emit_conditional_jump::<{Condition::NE}>(cb, *target); }, - Insn { op: Op::Jo, target, .. } => { - emit_conditional_jump::<{Condition::VS}>(cb, target.unwrap()); + Insn::Jo(target) => { + emit_conditional_jump::<{Condition::VS}>(cb, *target); }, - Insn { op: Op::IncrCounter, opnds, .. } => { - ldaddal(cb, opnds[1].into(), opnds[1].into(), opnds[0].into()); + Insn::IncrCounter { mem, value } => { + ldaddal(cb, value.into(), value.into(), mem.into()); }, - Insn { op: Op::Breakpoint, .. } => { + Insn::Breakpoint => { brk(cb, A64Opnd::None); }, - Insn { op: Op::CSelZ | Op::CSelE, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::EQ); + Insn::CSelZ { truthy, falsy, out } | + Insn::CSelE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::EQ); }, - Insn { op: Op::CSelNZ | Op::CSelNE, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::NE); + Insn::CSelNZ { truthy, falsy, out } | + Insn::CSelNE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::NE); }, - Insn { op: Op::CSelL, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::LT); + Insn::CSelL { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::LT); }, - Insn { op: Op::CSelLE, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::LE); + Insn::CSelLE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::LE); }, - Insn { op: Op::CSelG, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::GT); + Insn::CSelG { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GT); }, - Insn { op: Op::CSelGE, opnds, out, .. } => { - csel(cb, (*out).into(), opnds[0].into(), opnds[1].into(), Condition::GE); + Insn::CSelGE { truthy, falsy, out } => { + csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GE); } - Insn { op: Op::LiveReg, .. } => (), // just a reg alloc signal, no code - Insn { op: Op::PadEntryExit, .. } => { + Insn::LiveReg { .. } => (), // just a reg alloc signal, no code + Insn::PadEntryExit => { let jmp_len = 5 * 4; // Op::Jmp may emit 5 instructions while (cb.get_write_pos() - start_write_pos) < jmp_len { nop(cb); diff --git a/yjit/src/backend/ir.rs b/yjit/src/backend/ir.rs index cea8dfb227..fe525cf31d 100644 --- a/yjit/src/backend/ir.rs +++ b/yjit/src/backend/ir.rs @@ -24,155 +24,6 @@ pub const SP: Opnd = _SP; pub const C_ARG_OPNDS: [Opnd; 6] = _C_ARG_OPNDS; pub const C_RET_OPND: Opnd = _C_RET_OPND; -/// Instruction opcodes -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub enum Op -{ - // Add a comment into the IR at the point that this instruction is added. - // It won't have any impact on that actual compiled code. - Comment, - - // Add a label into the IR at the point that this instruction is added. - Label, - - // Mark a position in the generated code - PosMarker, - - // Bake a string directly into the instruction stream. - BakeString, - - // Add two operands together, and return the result as a new operand. This - // operand can then be used as the operand on another instruction. It - // accepts two operands, which can be of any type - // - // Under the hood when allocating registers, the IR will determine the most - // efficient way to get these values into memory. For example, if both - // operands are immediates, then it will load the first one into a register - // first with a mov instruction and then add them together. If one of them - // is a register, however, it will just perform a single add instruction. - Add, - - // This is the same as the OP_ADD instruction, except for subtraction. - Sub, - - // This is the same as the OP_ADD instruction, except that it performs the - // binary AND operation. - And, - - // This is the same as the OP_ADD instruction, except that it performs the - // binary OR operation. - Or, - - // This is the same as the OP_ADD instruction, except that it performs the - // binary XOR operation. - Xor, - - // Perform the NOT operation on an individual operand, and return the result - // as a new operand. This operand can then be used as the operand on another - // instruction. - Not, - - /// Shift a value right by a certain amount (signed). - RShift, - - /// Shift a value right by a certain amount (unsigned). - URShift, - - /// Shift a value left by a certain amount. - LShift, - - // - // Low-level instructions - // - - // A low-level instruction that loads a value into a register. - Load, - - // A low-level instruction that loads a value into a register and - // sign-extends it to a 64-bit value. - LoadSExt, - - // Low-level instruction to store a value to memory. - Store, - - // Load effective address - Lea, - - // Load effective address relative to the current instruction pointer. It - // accepts a single signed immediate operand. - LeaLabel, - - // A low-level mov instruction. It accepts two operands. - Mov, - - // Bitwise AND test instruction - Test, - - // Compare two operands - Cmp, - - // Unconditional jump to a branch target - Jmp, - - // Unconditional jump which takes a reg/mem address operand - JmpOpnd, - - // Low-level conditional jump instructions - Jl, - Jbe, - Je, - Jne, - Jz, - Jnz, - Jo, - - // Conditional select instructions - CSelZ, - CSelNZ, - CSelE, - CSelNE, - CSelL, - CSelLE, - CSelG, - CSelGE, - - // Push and pop registers to/from the C stack - CPush, - CPop, - CPopInto, - - // Push and pop all of the caller-save registers and the flags to/from the C - // stack - CPushAll, - CPopAll, - - // C function call with N arguments (variadic) - CCall, - - // C function return - CRet, - - // Atomically increment a counter - // Input: memory operand, increment value - // Produces no output - IncrCounter, - - // Trigger a debugger breakpoint - Breakpoint, - - /// Set up the frame stack as necessary per the architecture. - FrameSetup, - - /// Tear down the frame stack as necessary per the architecture. - FrameTeardown, - - /// Take a specific register. Signal the register allocator to not use it. - LiveReg, - - /// Pad nop instructions to accomodate Op::Jmp in case the block is invalidated. - PadEntryExit, -} - // Memory operand base #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum MemBase @@ -429,26 +280,170 @@ impl From<CodePtr> for Target { type PosMarkerFn = Box<dyn Fn(CodePtr)>; /// YJIT IR instruction -pub struct Insn -{ - // Opcode for the instruction - pub(super) op: Op, +pub enum Insn { + /// Add two operands together, and return the result as a new operand. + Add { left: Opnd, right: Opnd, out: Opnd }, + + /// This is the same as the OP_ADD instruction, except that it performs the + /// binary AND operation. + And { left: Opnd, right: Opnd, out: Opnd }, + + /// Bake a string directly into the instruction stream. + BakeString(String), + + // Trigger a debugger breakpoint + Breakpoint, + + /// Add a comment into the IR at the point that this instruction is added. + /// It won't have any impact on that actual compiled code. + Comment(String), + + /// Compare two operands + Cmp { left: Opnd, right: Opnd }, + + /// Pop a register from the C stack + CPop { out: Opnd }, + + /// Pop all of the caller-save registers and the flags from the C stack + CPopAll, + + /// Pop a register from the C stack and store it into another register + CPopInto(Opnd), + + /// Push a register onto the C stack + CPush(Opnd), + + /// Push all of the caller-save registers and the flags to the C stack + CPushAll, + + // C function call with N arguments (variadic) + CCall { opnds: Vec<Opnd>, target: Target, out: Opnd }, + + // C function return + CRet(Opnd), + + /// Conditionally select if equal + CSelE { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if greater + CSelG { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Conditionally select if greater or equal + CSelGE { truthy: Opnd, falsy: Opnd, out: Opnd }, - // Optional string for comments and labels - pub(super) text: Option<String>, + /// Conditionally select if less + CSelL { truthy: Opnd, falsy: Opnd, out: Opnd }, - // List of input operands/values - pub(super) opnds: Vec<Opnd>, + /// Conditionally select if less or equal + CSelLE { truthy: Opnd, falsy: Opnd, out: Opnd }, - // Output operand for this instruction - pub(super) out: Opnd, + /// Conditionally select if not equal + CSelNE { truthy: Opnd, falsy: Opnd, out: Opnd }, - // List of branch targets (branch instructions only) - pub(super) target: Option<Target>, + /// Conditionally select if not zero + CSelNZ { truthy: Opnd, falsy: Opnd, out: Opnd }, - // Callback to mark the position of this instruction - // in the generated code - pub(super) pos_marker: Option<PosMarkerFn>, + /// Conditionally select if zero + CSelZ { truthy: Opnd, falsy: Opnd, out: Opnd }, + + /// Set up the frame stack as necessary per the architecture. + FrameSetup, + + /// Tear down the frame stack as necessary per the architecture. + FrameTeardown, + + // Atomically increment a counter + // Input: memory operand, increment value + // Produces no output + IncrCounter { mem: Opnd, value: Opnd }, + + /// Jump if below or equal + Jbe(Target), + + /// Jump if equal + Je(Target), + + /// Jump if lower + Jl(Target), + + // Unconditional jump to a branch target + Jmp(Target), + + // Unconditional jump which takes a reg/mem address operand + JmpOpnd(Opnd), + + /// Jump if not equal + Jne(Target), + + /// Jump if not zero + Jnz(Target), + + /// Jump if overflow + Jo(Target), + + /// Jump if zero + Jz(Target), + + // Add a label into the IR at the point that this instruction is added. + Label(Target), + + // Load effective address relative to the current instruction pointer. It + // accepts a single signed immediate operand. + LeaLabel { target: Target, out: Opnd }, + + // Load effective address + Lea { opnd: Opnd, out: Opnd }, + + /// Take a specific register. Signal the register allocator to not use it. + LiveReg { opnd: Opnd, out: Opnd }, + + // A low-level instruction that loads a value into a register. + Load { opnd: Opnd, out: Opnd }, + + // A low-level instruction that loads a value into a register and + // sign-extends it to a 64-bit value. + LoadSExt { opnd: Opnd, out: Opnd }, + + /// Shift a value left by a certain amount. + LShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // A low-level mov instruction. It accepts two operands. + Mov { dest: Opnd, src: Opnd }, + + // Perform the NOT operation on an individual operand, and return the result + // as a new operand. This operand can then be used as the operand on another + // instruction. + Not { opnd: Opnd, out: Opnd }, + + // This is the same as the OP_ADD instruction, except that it performs the + // binary OR operation. + Or { left: Opnd, right: Opnd, out: Opnd }, + + /// Pad nop instructions to accomodate Op::Jmp in case the block is + /// invalidated. + PadEntryExit, + + // Mark a position in the generated code + PosMarker(PosMarkerFn), + + /// Shift a value right by a certain amount (signed). + RShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // Low-level instruction to store a value to memory. + Store { dest: Opnd, src: Opnd }, + + // This is the same as the OP_ADD instruction, except for subtraction. + Sub { left: Opnd, right: Opnd, out: Opnd }, + + // Bitwise AND test instruction + Test { left: Opnd, right: Opnd }, + + /// Shift a value right by a certain amount (unsigned). + URShift { opnd: Opnd, shift: Opnd, out: Opnd }, + + // This is the same as the OP_ADD instruction, except that it performs the + // binary XOR operation. + Xor { left: Opnd, right: Opnd, out: Opnd } } impl Insn { @@ -464,34 +459,92 @@ impl Insn { InsnOpndMutIterator::new(self) } + /// Returns a string that describes which operation this instruction is + /// performing. This is used for debugging. + fn op(&self) -> &'static str { + match self { + Insn::Add { .. } => "Add", + Insn::And { .. } => "And", + Insn::BakeString(_) => "BakeString", + Insn::Breakpoint => "Breakpoint", + Insn::Comment(_) => "Comment", + Insn::Cmp { .. } => "Cmp", + Insn::CPop { .. } => "CPop", + Insn::CPopAll => "CPopAll", + Insn::CPopInto(_) => "CPopInto", + Insn::CPush(_) => "CPush", + Insn::CPushAll => "CPushAll", + Insn::CCall { .. } => "CCall", + Insn::CRet(_) => "CRet", + Insn::CSelE { .. } => "CSelE", + Insn::CSelG { .. } => "CSelG", + Insn::CSelGE { .. } => "CSelGE", + Insn::CSelL { .. } => "CSelL", + Insn::CSelLE { .. } => "CSelLE", + Insn::CSelNE { .. } => "CSelNE", + Insn::CSelNZ { .. } => "CSelNZ", + Insn::CSelZ { .. } => "CSelZ", + Insn::FrameSetup => "FrameSetup", + Insn::FrameTeardown => "FrameTeardown", + Insn::IncrCounter { .. } => "IncrCounter", + Insn::Jbe(_) => "Jbe", + Insn::Je(_) => "Je", + Insn::Jl(_) => "Jl", + Insn::Jmp(_) => "Jmp", + Insn::JmpOpnd(_) => "JmpOpnd", + Insn::Jne(_) => "Jne", + Insn::Jnz(_) => "Jnz", + Insn::Jo(_) => "Jo", + Insn::Jz(_) => "Jz", + Insn::Label(_) => "Label", + Insn::LeaLabel { .. } => "LeaLabel", + Insn::Lea { .. } => "Lea", + Insn::LiveReg { .. } => "LiveReg", + Insn::Load { .. } => "Load", + Insn::LoadSExt { .. } => "LoadSExt", + Insn::LShift { .. } => "LShift", + Insn::Mov { .. } => "Mov", + Insn::Not { .. } => "Not", + Insn::Or { .. } => "Or", + Insn::PadEntryExit => "PadEntryExit", + Insn::PosMarker(_) => "PosMarker", + Insn::RShift { .. } => "RShift", + Insn::Store { .. } => "Store", + Insn::Sub { .. } => "Sub", + Insn::Test { .. } => "Test", + Insn::URShift { .. } => "URShift", + Insn::Xor { .. } => "Xor" + } + } + /// Return a non-mutable reference to the out operand for this instruction /// if it has one. pub fn out_opnd(&self) -> Option<&Opnd> { match self { - Insn { op: Op::Add, out, .. } | - Insn { op: Op::And, out, .. } | - Insn { op: Op::CCall, out, .. } | - Insn { op: Op::CPop, out, .. } | - Insn { op: Op::CSelE, out, .. } | - Insn { op: Op::CSelG, out, .. } | - Insn { op: Op::CSelGE, out, .. } | - Insn { op: Op::CSelL, out, .. } | - Insn { op: Op::CSelLE, out, .. } | - Insn { op: Op::CSelNE, out, .. } | - Insn { op: Op::CSelNZ, out, .. } | - Insn { op: Op::CSelZ, out, .. } | - Insn { op: Op::Lea, out, .. } | - Insn { op: Op::LeaLabel, out, .. } | - Insn { op: Op::LiveReg, out, .. } | - Insn { op: Op::Load, out, .. } | - Insn { op: Op::LoadSExt, out, .. } | - Insn { op: Op::LShift, out, .. } | - Insn { op: Op::Not, out, .. } | - Insn { op: Op::Or, out, .. } | - Insn { op: Op::RShift, out, .. } | - Insn { op: Op::Sub, out, .. } | - Insn { op: Op::URShift, out, .. } | - Insn { op: Op::Xor, out, .. } => Some(out), + Insn::Add { out, .. } | + Insn::And { out, .. } | + Insn::CCall { out, .. } | + Insn::CPop { out, .. } | + Insn::CSelE { out, .. } | + Insn::CSelG { out, .. } | + Insn::CSelGE { out, .. } | + Insn::CSelL { out, .. } | + Insn::CSelLE { out, .. } | + Insn::CSelNE { out, .. } | + Insn::CSelNZ { out, .. } | + Insn::CSelZ { out, .. } | + Insn::Lea { out, .. } | + Insn::LeaLabel { out, .. } | + Insn::LiveReg { out, .. } | + Insn::Load { out, .. } | + Insn::LoadSExt { out, .. } | + Insn::LShift { out, .. } | + Insn::Not { out, .. } | + Insn::Or { out, .. } | + Insn::RShift { out, .. } | + Insn::Sub { out, .. } | + Insn::URShift { out, .. } | + Insn::Xor { out, .. } => Some(out), _ => None } } @@ -500,30 +553,55 @@ impl Insn { /// has one. pub fn out_opnd_mut(&mut self) -> Option<&mut Opnd> { match self { - Insn { op: Op::Add, out, .. } | - Insn { op: Op::And, out, .. } | - Insn { op: Op::CCall, out, .. } | - Insn { op: Op::CPop, out, .. } | - Insn { op: Op::CSelE, out, .. } | - Insn { op: Op::CSelG, out, .. } | - Insn { op: Op::CSelGE, out, .. } | - Insn { op: Op::CSelL, out, .. } | - Insn { op: Op::CSelLE, out, .. } | - Insn { op: Op::CSelNE, out, .. } | - Insn { op: Op::CSelNZ, out, .. } | - Insn { op: Op::CSelZ, out, .. } | - Insn { op: Op::Lea, out, .. } | - Insn { op: Op::LeaLabel, out, .. } | - Insn { op: Op::LiveReg, out, .. } | - Insn { op: Op::Load, out, .. } | - Insn { op: Op::LoadSExt, out, .. } | - Insn { op: Op::LShift, out, .. } | - Insn { op: Op::Not, out, .. } | - Insn { op: Op::Or, out, .. } | - Insn { op: Op::RShift, out, .. } | - Insn { op: Op::Sub, out, .. } | - Insn { op: Op::URShift, out, .. } | - Insn { op: Op::Xor, out, .. } => Some(out), + Insn::Add { out, .. } | + Insn::And { out, .. } | + Insn::CCall { out, .. } | + Insn::CPop { out, .. } | + Insn::CSelE { out, .. } | + Insn::CSelG { out, .. } | + Insn::CSelGE { out, .. } | + Insn::CSelL { out, .. } | + Insn::CSelLE { out, .. } | + Insn::CSelNE { out, .. } | + Insn::CSelNZ { out, .. } | + Insn::CSelZ { out, .. } | + Insn::Lea { out, .. } | + Insn::LeaLabel { out, .. } | + Insn::LiveReg { out, .. } | + Insn::Load { out, .. } | + Insn::LoadSExt { out, .. } | + Insn::LShift { out, .. } | + Insn::Not { out, .. } | + Insn::Or { out, .. } | + Insn::RShift { out, .. } | + Insn::Sub { out, .. } | + Insn::URShift { out, .. } | + Insn::Xor { out, .. } => Some(out), + _ => None + } + } + + /// Returns the target for this instruction if there is one. + pub fn target(&self) -> Option<&Target> { + match self { + Insn::Jbe(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::Jz(target) | + Insn::LeaLabel { target, .. } => Some(target), + _ => None + } + } + + /// Returns the text associated with this instruction if there is some. + pub fn text(&self) -> Option<&String> { + match self { + Insn::BakeString(text) | + Insn::Comment(text) => Some(text), _ => None } } @@ -547,77 +625,77 @@ impl<'a> Iterator for InsnOpndIterator<'a> { fn next(&mut self) -> Option<Self::Item> { match self.insn { - Insn { op: Op::BakeString, .. } | - Insn { op: Op::Breakpoint, .. } | - Insn { op: Op::Comment, .. } | - Insn { op: Op::CPop, .. } | - Insn { op: Op::CPopAll, .. } | - Insn { op: Op::CPushAll, .. } | - Insn { op: Op::FrameSetup, .. } | - Insn { op: Op::FrameTeardown, .. } | - Insn { op: Op::Jbe, .. } | - Insn { op: Op::Je, .. } | - Insn { op: Op::Jl, .. } | - Insn { op: Op::Jmp, .. } | - Insn { op: Op::Jne, .. } | - Insn { op: Op::Jnz, .. } | - Insn { op: Op::Jo, .. } | - Insn { op: Op::Jz, .. } | - Insn { op: Op::Label, .. } | - Insn { op: Op::LeaLabel, .. } | - Insn { op: Op::PadEntryExit, .. } | - Insn { op: Op::PosMarker, .. } => None, - Insn { op: Op::CPopInto, opnds, .. } | - Insn { op: Op::CPush, opnds, .. } | - Insn { op: Op::CRet, opnds, .. } | - Insn { op: Op::JmpOpnd, opnds, .. } | - Insn { op: Op::Lea, opnds, .. } | - Insn { op: Op::LiveReg, opnds, .. } | - Insn { op: Op::Load, opnds, .. } | - Insn { op: Op::LoadSExt, opnds, .. } | - Insn { op: Op::Not, opnds, .. } => { + Insn::BakeString(_) | + Insn::Breakpoint | + Insn::Comment(_) | + Insn::CPop { .. } | + Insn::CPopAll | + Insn::CPushAll | + Insn::FrameSetup | + Insn::FrameTeardown | + Insn::Jbe(_) | + Insn::Je(_) | + Insn::Jl(_) | + Insn::Jmp(_) | + Insn::Jne(_) | + Insn::Jnz(_) | + Insn::Jo(_) | + Insn::Jz(_) | + Insn::Label(_) | + Insn::LeaLabel { .. } | + Insn::PadEntryExit | + Insn::PosMarker(_) => None, + Insn::CPopInto(opnd) | + Insn::CPush(opnd) | + Insn::CRet(opnd) | + Insn::JmpOpnd(opnd) | + Insn::Lea { opnd, .. } | + Insn::LiveReg { opnd, .. } | + Insn::Load { opnd, .. } | + Insn::LoadSExt { opnd, .. } | + Insn::Not { opnd, .. } => { match self.idx { 0 => { self.idx += 1; - Some(&opnds[0]) + Some(&opnd) }, _ => None } }, - Insn { op: Op::Add, opnds, .. } | - Insn { op: Op::And, opnds, .. } | - Insn { op: Op::Cmp, opnds, .. } | - Insn { op: Op::CSelE, opnds, .. } | - Insn { op: Op::CSelG, opnds, .. } | - Insn { op: Op::CSelGE, opnds, .. } | - Insn { op: Op::CSelL, opnds, .. } | - Insn { op: Op::CSelLE, opnds, .. } | - Insn { op: Op::CSelNE, opnds, .. } | - Insn { op: Op::CSelNZ, opnds, .. } | - Insn { op: Op::CSelZ, opnds, .. } | - Insn { op: Op::IncrCounter, opnds, .. } | - Insn { op: Op::LShift, opnds, .. } | - Insn { op: Op::Mov, opnds, .. } | - Insn { op: Op::Or, opnds, .. } | - Insn { op: Op::RShift, opnds, .. } | - Insn { op: Op::Store, opnds, .. } | - Insn { op: Op::Sub, opnds, .. } | - Insn { op: Op::Test, opnds, .. } | - Insn { op: Op::URShift, opnds, .. } | - Insn { op: Op::Xor, opnds, .. } => { + Insn::Add { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::And { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::Cmp { left: opnd0 @ _, right: opnd1 @ _ } | + Insn::CSelE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelG { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelGE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelL { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelLE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelNE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelNZ { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelZ { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::IncrCounter { mem: opnd0 @ _, value: opnd1 @ _, .. } | + Insn::LShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Mov { dest: opnd0 @ _, src: opnd1 @ _ } | + Insn::Or { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::RShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Store { dest: opnd0 @ _, src: opnd1 @ _ } | + Insn::Sub { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::Test { left: opnd0 @ _, right: opnd1 @ _ } | + Insn::URShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Xor { left: opnd0 @ _, right: opnd1 @ _, .. } => { match self.idx { 0 => { self.idx += 1; - Some(&opnds[0]) + Some(&opnd0) } 1 => { self.idx += 1; - Some(&opnds[1]) + Some(&opnd1) } _ => None } }, - Insn { op: Op::CCall, opnds, .. } => { + Insn::CCall { opnds, .. } => { if self.idx < opnds.len() { let opnd = &opnds[self.idx]; self.idx += 1; @@ -643,77 +721,77 @@ impl<'a> InsnOpndMutIterator<'a> { pub(super) fn next(&mut self) -> Option<&mut Opnd> { match self.insn { - Insn { op: Op::BakeString, .. } | - Insn { op: Op::Breakpoint, .. } | - Insn { op: Op::Comment, .. } | - Insn { op: Op::CPop, .. } | - Insn { op: Op::CPopAll, .. } | - Insn { op: Op::CPushAll, .. } | - Insn { op: Op::FrameSetup, .. } | - Insn { op: Op::FrameTeardown, .. } | - Insn { op: Op::Jbe, .. } | - Insn { op: Op::Je, .. } | - Insn { op: Op::Jl, .. } | - Insn { op: Op::Jmp, .. } | - Insn { op: Op::Jne, .. } | - Insn { op: Op::Jnz, .. } | - Insn { op: Op::Jo, .. } | - Insn { op: Op::Jz, .. } | - Insn { op: Op::Label, .. } | - Insn { op: Op::LeaLabel, .. } | - Insn { op: Op::PadEntryExit, .. } | - Insn { op: Op::PosMarker, .. } => None, - Insn { op: Op::CPopInto, opnds, .. } | - Insn { op: Op::CPush, opnds, .. } | - Insn { op: Op::CRet, opnds, .. } | - Insn { op: Op::JmpOpnd, opnds, .. } | - Insn { op: Op::Lea, opnds, .. } | - Insn { op: Op::LiveReg, opnds, .. } | - Insn { op: Op::Load, opnds, .. } | - Insn { op: Op::LoadSExt, opnds, .. } | - Insn { op: Op::Not, opnds, .. } => { + Insn::BakeString(_) | + Insn::Breakpoint | + Insn::Comment(_) | + Insn::CPop { .. } | + Insn::CPopAll | + Insn::CPushAll | + Insn::FrameSetup | + Insn::FrameTeardown | + Insn::Jbe(_) | + Insn::Je(_) | + Insn::Jl(_) | + Insn::Jmp(_) | + Insn::Jne(_) | + Insn::Jnz(_) | + Insn::Jo(_) | + Insn::Jz(_) | + Insn::Label(_) | + Insn::LeaLabel { .. } | + Insn::PadEntryExit | + Insn::PosMarker(_) => None, + Insn::CPopInto(opnd) | + Insn::CPush(opnd) | + Insn::CRet(opnd) | + Insn::JmpOpnd(opnd) | + Insn::Lea { opnd, .. } | + Insn::LiveReg { opnd, .. } | + Insn::Load { opnd, .. } | + Insn::LoadSExt { opnd, .. } | + Insn::Not { opnd, .. } => { match self.idx { 0 => { self.idx += 1; - Some(&mut opnds[0]) + Some(opnd) }, _ => None } }, - Insn { op: Op::Add, opnds, .. } | - Insn { op: Op::And, opnds, .. } | - Insn { op: Op::Cmp, opnds, .. } | - Insn { op: Op::CSelE, opnds, .. } | - Insn { op: Op::CSelG, opnds, .. } | - Insn { op: Op::CSelGE, opnds, .. } | - Insn { op: Op::CSelL, opnds, .. } | - Insn { op: Op::CSelLE, opnds, .. } | - Insn { op: Op::CSelNE, opnds, .. } | - Insn { op: Op::CSelNZ, opnds, .. } | - Insn { op: Op::CSelZ, opnds, .. } | - Insn { op: Op::IncrCounter, opnds, .. } | - Insn { op: Op::LShift, opnds, .. } | - Insn { op: Op::Mov, opnds, .. } | - Insn { op: Op::Or, opnds, .. } | - Insn { op: Op::RShift, opnds, .. } | - Insn { op: Op::Store, opnds, .. } | - Insn { op: Op::Sub, opnds, .. } | - Insn { op: Op::Test, opnds, .. } | - Insn { op: Op::URShift, opnds, .. } | - Insn { op: Op::Xor, opnds, .. } => { + Insn::Add { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::And { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::Cmp { left: opnd0 @ _, right: opnd1 @ _ } | + Insn::CSelE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelG { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelGE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelL { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelLE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelNE { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelNZ { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::CSelZ { truthy: opnd0 @ _, falsy: opnd1 @ _, .. } | + Insn::IncrCounter { mem: opnd0 @ _, value: opnd1 @ _, .. } | + Insn::LShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Mov { dest: opnd0 @ _, src: opnd1 @ _ } | + Insn::Or { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::RShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Store { dest: opnd0 @ _, src: opnd1 @ _ } | + Insn::Sub { left: opnd0 @ _, right: opnd1 @ _, .. } | + Insn::Test { left: opnd0 @ _, right: opnd1 @ _ } | + Insn::URShift { opnd: opnd0 @ _, shift: opnd1 @ _, .. } | + Insn::Xor { left: opnd0 @ _, right: opnd1 @ _, .. } => { match self.idx { 0 => { self.idx += 1; - Some(&mut opnds[0]) + Some(opnd0) } 1 => { self.idx += 1; - Some(&mut opnds[1]) + Some(opnd1) } _ => None } }, - Insn { op: Op::CCall, opnds, .. } => { + Insn::CCall { opnds, .. } => { if self.idx < opnds.len() { let opnd = &mut opnds[self.idx]; self.idx += 1; @@ -728,7 +806,7 @@ impl<'a> InsnOpndMutIterator<'a> { impl fmt::Debug for Insn { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "{:?}(", self.op)?; + write!(fmt, "{}(", self.op())?; // Print list of operands let mut opnd_iter = self.opnd_iter(); @@ -741,10 +819,10 @@ impl fmt::Debug for Insn { write!(fmt, ")")?; // Print text, target, and pos if they are present - if let Some(text) = &self.text { + if let Some(text) = self.text() { write!(fmt, " {text:?}")? } - if let Some(target) = self.target { + if let Some(target) = self.target() { write!(fmt, " target={target:?}")?; } @@ -814,23 +892,6 @@ impl Assembler self.live_ranges.push(insn_idx); } - /// Append an instruction to the list by creating a new instruction from the - /// component parts given to this function. This will also create a new - /// output operand from the given operands for the new instruction. - pub(super) fn push_insn_parts( - &mut self, - op: Op, - opnds: Vec<Opnd>, - target: Option<Target>, - text: Option<String>, - pos_marker: Option<PosMarkerFn> - ) -> Opnd - { - let out = self.next_opnd_out(Opnd::match_num_bits(&opnds)); - self.push_insn(Insn { op, text, opnds, out, target, pos_marker }); - out - } - /// Create a new label instance that we can jump to pub fn new_label(&mut self, name: &str) -> Target { @@ -841,23 +902,6 @@ impl Assembler Target::Label(label_idx) } - /// Add a label at the current position - pub fn write_label(&mut self, label: Target) - { - assert!(label.unwrap_label_idx() < self.label_names.len()); - - let insn = Insn { - op: Op::Label, - text: None, - opnds: vec![], - out: Opnd::None, - target: Some(label), - pos_marker: None, - }; - self.insns.push(insn); - self.live_ranges.push(self.insns.len()); - } - /// Sets the out field on the various instructions that require allocated /// registers because their output is used as the operand on a subsequent /// instruction. This is our implementation of the linear scan algorithm. @@ -928,7 +972,7 @@ impl Assembler if let Some(Opnd::Reg(reg)) = asm.insns[start_index].out_opnd() { dealloc_reg(&mut pool, ®s, reg); } else { - unreachable!("no register allocated for insn {:?}", insn.op); + unreachable!("no register allocated for insn {:?}", insn); } } } @@ -937,7 +981,7 @@ impl Assembler } // C return values need to be mapped to the C return register - if insn.op == Op::CCall { + if matches!(insn, Insn::CCall { .. }) { assert_eq!(pool, 0, "register lives past C function call"); } @@ -958,7 +1002,7 @@ impl Assembler let mut out_reg: Option<Reg> = None; // C return values need to be mapped to the C return register - if insn.op == Op::CCall { + if matches!(insn, Insn::CCall { .. }) { out_reg = Some(take_reg(&mut pool, ®s, &C_RET_REG)); } @@ -983,9 +1027,9 @@ impl Assembler // already allocated. if out_reg.is_none() { out_reg = match &insn { - Insn { op: Op::LiveReg, opnds, .. } => { + Insn::LiveReg { opnd, .. } => { // Allocate a specific register - let reg = opnds[0].unwrap_reg(); + let reg = opnd.unwrap_reg(); Some(take_reg(&mut pool, ®s, ®)) }, _ => { @@ -1090,9 +1134,13 @@ impl AssemblerDrainingIterator { /// Returns the next instruction in the list with the indices corresponding /// to the next list of instructions. pub fn next_mapped(&mut self) -> Option<(usize, Insn)> { - self.next_unmapped().map(|(index, insn)| { - let opnds = insn.opnd_iter().map(|opnd| opnd.map_index(&self.indices)).collect(); - (index, Insn { opnds, ..insn }) + self.next_unmapped().map(|(index, mut insn)| { + let mut opnd_iter = insn.opnd_iter_mut(); + while let Some(opnd) = opnd_iter.next() { + *opnd = opnd.map_index(&self.indices); + } + + (index, insn) }) } @@ -1167,273 +1215,279 @@ impl Assembler { #[must_use] pub fn add(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::Add, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Add { left, right, out }); out } #[must_use] pub fn and(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::And, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::And { left, right, out }); out } pub fn bake_string(&mut self, text: &str) { - self.push_insn(Insn { op: Op::BakeString, opnds: vec![], out: Opnd::None, text: Some(text.to_string()), target: None, pos_marker: None }); + self.push_insn(Insn::BakeString(text.to_string())); } pub fn breakpoint(&mut self) { - self.push_insn(Insn { op: Op::Breakpoint, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Breakpoint); } #[must_use] pub fn ccall(&mut self, fptr: *const u8, opnds: Vec<Opnd>) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&opnds)); - self.push_insn(Insn { op: Op::CCall, opnds, out, text: None, target: Some(Target::FunPtr(fptr)), pos_marker: None }); + self.push_insn(Insn::CCall { target: Target::FunPtr(fptr), opnds, out }); out } pub fn cmp(&mut self, left: Opnd, right: Opnd) { - self.push_insn(Insn { op: Op::Cmp, opnds: vec![left, right], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Cmp { left, right }); } pub fn comment(&mut self, text: &str) { - self.push_insn(Insn { op: Op::Comment, opnds: vec![], out: Opnd::None, text: Some(text.to_string()), target: None, pos_marker: None }); + self.push_insn(Insn::Comment(text.to_string())); } #[must_use] pub fn cpop(&mut self) -> Opnd { let out = self.next_opnd_out(Opnd::DEFAULT_NUM_BITS); - self.push_insn(Insn { op: Op::CPop, opnds: vec![], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPop { out }); out } pub fn cpop_all(&mut self) { - self.push_insn(Insn { op: Op::CPopAll, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPopAll); } pub fn cpop_into(&mut self, opnd: Opnd) { - self.push_insn(Insn { op: Op::CPopInto, opnds: vec![opnd], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPopInto(opnd)); } pub fn cpush(&mut self, opnd: Opnd) { - self.push_insn(Insn { op: Op::CPush, opnds: vec![opnd], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPush(opnd)); } pub fn cpush_all(&mut self) { - self.push_insn(Insn { op: Op::CPushAll, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CPushAll); } pub fn cret(&mut self, opnd: Opnd) { - self.push_insn(Insn { op: Op::CRet, opnds: vec![opnd], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CRet(opnd)); } #[must_use] pub fn csel_e(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelE, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelE { truthy, falsy, out }); out } #[must_use] pub fn csel_g(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelG, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelG { truthy, falsy, out }); out } #[must_use] pub fn csel_ge(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelGE, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelGE { truthy, falsy, out }); out } #[must_use] pub fn csel_l(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelL, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelL { truthy, falsy, out }); out } #[must_use] pub fn csel_le(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelLE, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelLE { truthy, falsy, out }); out } #[must_use] pub fn csel_ne(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelNE, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelNE { truthy, falsy, out }); out } #[must_use] pub fn csel_nz(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelNZ, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelNZ { truthy, falsy, out }); out } #[must_use] pub fn csel_z(&mut self, truthy: Opnd, falsy: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[truthy, falsy])); - self.push_insn(Insn { op: Op::CSelZ, opnds: vec![truthy, falsy], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::CSelZ { truthy, falsy, out }); out } pub fn frame_setup(&mut self) { - self.push_insn(Insn { op: Op::FrameSetup, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::FrameSetup); } pub fn frame_teardown(&mut self) { - self.push_insn(Insn { op: Op::FrameTeardown, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::FrameTeardown); } pub fn incr_counter(&mut self, mem: Opnd, value: Opnd) { - self.push_insn(Insn { op: Op::IncrCounter, opnds: vec![mem, value], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::IncrCounter { mem, value }); } pub fn jbe(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jbe, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jbe(target)); } pub fn je(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Je, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Je(target)); } pub fn jl(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jl, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jl(target)); } pub fn jmp(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jmp, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jmp(target)); } pub fn jmp_opnd(&mut self, opnd: Opnd) { - self.push_insn(Insn { op: Op::JmpOpnd, opnds: vec![opnd], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::JmpOpnd(opnd)); } pub fn jne(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jne, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jne(target)); } pub fn jnz(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jnz, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jnz(target)); } pub fn jo(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jo, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jo(target)); } pub fn jz(&mut self, target: Target) { - self.push_insn(Insn { op: Op::Jz, opnds: vec![], out: Opnd::None, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::Jz(target)); } #[must_use] pub fn lea(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::Lea, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Lea { opnd, out }); out } #[must_use] pub fn lea_label(&mut self, target: Target) -> Opnd { let out = self.next_opnd_out(Opnd::DEFAULT_NUM_BITS); - self.push_insn(Insn { op: Op::LeaLabel, opnds: vec![], out, text: None, target: Some(target), pos_marker: None }); + self.push_insn(Insn::LeaLabel { target, out }); out } #[must_use] pub fn live_reg_opnd(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::LiveReg, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::LiveReg { opnd, out }); out } #[must_use] pub fn load(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::Load, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Load { opnd, out }); out } #[must_use] pub fn load_sext(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::LoadSExt, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::LoadSExt { opnd, out }); out } #[must_use] pub fn lshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd, shift])); - self.push_insn(Insn { op: Op::LShift, opnds: vec![opnd, shift], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::LShift { opnd, shift, out }); out } pub fn mov(&mut self, dest: Opnd, src: Opnd) { - self.push_insn(Insn { op: Op::Mov, opnds: vec![dest, src], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Mov { dest, src }); } #[must_use] pub fn not(&mut self, opnd: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn { op: Op::Not, opnds: vec![opnd], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Not { opnd, out }); out } #[must_use] pub fn or(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::Or, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Or { left, right, out }); out } pub fn pad_entry_exit(&mut self) { - self.push_insn(Insn { op: Op::PadEntryExit, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::PadEntryExit); } //pub fn pos_marker<F: FnMut(CodePtr)>(&mut self, marker_fn: F) pub fn pos_marker(&mut self, marker_fn: impl Fn(CodePtr) + 'static) { - self.push_insn(Insn { op: Op::PosMarker, opnds: vec![], out: Opnd::None, text: None, target: None, pos_marker: Some(Box::new(marker_fn)) }); + self.push_insn(Insn::PosMarker(Box::new(marker_fn))); } #[must_use] pub fn rshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd, shift])); - self.push_insn(Insn { op: Op::RShift, opnds: vec![opnd, shift], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::RShift { opnd, shift, out }); out } pub fn store(&mut self, dest: Opnd, src: Opnd) { - self.push_insn(Insn { op: Op::Store, opnds: vec![dest, src], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Store { dest, src }); } #[must_use] pub fn sub(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::Sub, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Sub { left, right, out }); out } pub fn test(&mut self, left: Opnd, right: Opnd) { - self.push_insn(Insn { op: Op::Test, opnds: vec![left, right], out: Opnd::None, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Test { left, right }); } #[must_use] pub fn urshift(&mut self, opnd: Opnd, shift: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[opnd, shift])); - self.push_insn(Insn { op: Op::URShift, opnds: vec![opnd, shift], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::URShift { opnd, shift, out }); out } + /// Add a label at the current position + pub fn write_label(&mut self, target: Target) { + assert!(target.unwrap_label_idx() < self.label_names.len()); + self.push_insn(Insn::Label(target)); + } + #[must_use] pub fn xor(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.next_opnd_out(Opnd::match_num_bits(&[left, right])); - self.push_insn(Insn { op: Op::Xor, opnds: vec![left, right], out, text: None, target: None, pos_marker: None }); + self.push_insn(Insn::Xor { left, right, out }); out } } @@ -1444,7 +1498,7 @@ mod tests { #[test] fn test_opnd_iter() { - let insn = Insn { op: Op::Add, opnds: vec![Opnd::None, Opnd::None], out: Opnd::None, text: None, target: None, pos_marker: None }; + let insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; let mut opnd_iter = insn.opnd_iter(); assert!(matches!(opnd_iter.next(), Some(Opnd::None))); @@ -1455,7 +1509,7 @@ mod tests { #[test] fn test_opnd_iter_mut() { - let mut insn = Insn { op: Op::Add, opnds: vec![Opnd::None, Opnd::None], out: Opnd::None, text: None, target: None, pos_marker: None }; + let mut insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; let mut opnd_iter = insn.opnd_iter_mut(); assert!(matches!(opnd_iter.next(), Some(Opnd::None))); diff --git a/yjit/src/backend/tests.rs b/yjit/src/backend/tests.rs index b89b7eb648..08e8849b4d 100644 --- a/yjit/src/backend/tests.rs +++ b/yjit/src/backend/tests.rs @@ -315,9 +315,9 @@ fn test_draining_iterator() { while let Some((index, insn)) = iter.next_unmapped() { match index { - 0 => assert_eq!(insn.op, Op::Load), - 1 => assert_eq!(insn.op, Op::Store), - 2 => assert_eq!(insn.op, Op::Add), + 0 => assert!(matches!(insn, Insn::Load { .. })), + 1 => assert!(matches!(insn, Insn::Store { .. })), + 2 => assert!(matches!(insn, Insn::Add { .. })), _ => panic!("Unexpected instruction index"), }; } @@ -337,7 +337,7 @@ fn test_lookback_iterator() { if index > 0 { let opnd_iter = iter.get_previous().unwrap().opnd_iter(); assert_eq!(opnd_iter.take(1).next(), Some(&Opnd::None)); - assert_eq!(insn.op, Op::Store); + assert!(matches!(insn, Insn::Store { .. })); } } } diff --git a/yjit/src/backend/x86_64/mod.rs b/yjit/src/backend/x86_64/mod.rs index 0c994144d0..bda7dc4c06 100644 --- a/yjit/src/backend/x86_64/mod.rs +++ b/yjit/src/backend/x86_64/mod.rs @@ -69,6 +69,13 @@ impl From<Opnd> for X86Opnd { } } +/// Also implement going from a reference to an operand for convenience. +impl From<&Opnd> for X86Opnd { + fn from(opnd: &Opnd) -> Self { + X86Opnd::from(*opnd) + } +} + impl Assembler { // A special scratch register for intermediate processing. @@ -96,11 +103,47 @@ impl Assembler /// Split IR instructions for the x86 platform fn x86_split(mut self) -> Assembler { + fn split_arithmetic_opnds(asm: &mut Assembler, live_ranges: &Vec<usize>, index: usize, unmapped_opnds: &Vec<Opnd>, left: &Opnd, right: &Opnd) -> (Opnd, Opnd) { + match (unmapped_opnds[0], unmapped_opnds[1]) { + (Opnd::Mem(_), Opnd::Mem(_)) => { + (asm.load(*left), asm.load(*right)) + }, + (Opnd::Mem(_), Opnd::UImm(value)) => { + // 32-bit values will be sign-extended + if imm_num_bits(value as i64) > 32 { + (asm.load(*left), asm.load(*right)) + } else { + (asm.load(*left), *right) + } + }, + (Opnd::Mem(_), Opnd::Imm(value)) => { + if imm_num_bits(value) > 32 { + (asm.load(*left), asm.load(*right)) + } else { + (asm.load(*left), *right) + } + }, + // Instruction output whose live range spans beyond this instruction + (Opnd::InsnOut { idx, .. }, _) => { + if live_ranges[idx] > index { + (asm.load(*left), *right) + } else { + (*left, *right) + } + }, + // We have to load memory operands to avoid corrupting them + (Opnd::Mem(_) | Opnd::Reg(_), _) => { + (asm.load(*left), *right) + }, + _ => (*left, *right) + } + } + let live_ranges: Vec<usize> = take(&mut self.live_ranges); let mut asm = Assembler::new_with_label_names(take(&mut self.label_names)); let mut iterator = self.into_draining_iter(); - while let Some((index, insn)) = iterator.next_unmapped() { + while let Some((index, mut insn)) = iterator.next_unmapped() { // When we're iterating through the instructions with x86_split, we // need to know the previous live ranges in order to tell if a // register lasts beyond the current instruction. So instead of @@ -122,8 +165,15 @@ impl Assembler // - Most instructions can't be encoded with 64-bit immediates. // - We look for Op::Load specifically when emiting to keep GC'ed // VALUEs alive. This is a sort of canonicalization. - let mapped_opnds: Vec<Opnd> = insn.opnd_iter().map(|opnd| { - if insn.op == Op::Load { + let mut unmapped_opnds: Vec<Opnd> = vec![]; + + let is_load = matches!(insn, Insn::Load { .. }); + let mut opnd_iter = insn.opnd_iter_mut(); + + while let Some(opnd) = opnd_iter.next() { + unmapped_opnds.push(*opnd); + + *opnd = if is_load { iterator.map_opnd(*opnd) } else if let Opnd::Value(value) = opnd { // Since mov(mem64, imm32) sign extends, as_i64() makes sure @@ -136,130 +186,137 @@ impl Assembler } else { iterator.map_opnd(*opnd) } - }).collect(); + } - match insn { - Insn { op: Op::Add | Op::Sub | Op::And | Op::Cmp | Op::Or | Op::Test | Op::Xor, opnds, target, text, pos_marker, .. } => { - let (opnd0, opnd1) = match (opnds[0], opnds[1]) { - (Opnd::Mem(_), Opnd::Mem(_)) => { - (asm.load(mapped_opnds[0]), asm.load(mapped_opnds[1])) - }, - (Opnd::Mem(_), Opnd::UImm(value)) => { - // 32-bit values will be sign-extended - if imm_num_bits(value as i64) > 32 { - (asm.load(mapped_opnds[0]), asm.load(mapped_opnds[1])) - } else { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - } - }, - (Opnd::Mem(_), Opnd::Imm(value)) => { - if imm_num_bits(value) > 32 { - (asm.load(mapped_opnds[0]), asm.load(mapped_opnds[1])) - } else { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - } - }, - // Instruction output whose live range spans beyond this instruction - (Opnd::InsnOut { idx, .. }, _) => { - if live_ranges[idx] > index { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - } else { - (mapped_opnds[0], mapped_opnds[1]) - } - }, - // We have to load memory operands to avoid corrupting them - (Opnd::Mem(_) | Opnd::Reg(_), _) => { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - }, - _ => (mapped_opnds[0], mapped_opnds[1]) - }; + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } => { + let (split_left, split_right) = split_arithmetic_opnds(&mut asm, &live_ranges, index, &unmapped_opnds, left, right); + + *left = split_left; + *right = split_right; + *out = asm.next_opnd_out(Opnd::match_num_bits(&[*left, *right])); - asm.push_insn_parts(insn.op, vec![opnd0, opnd1], target, text, pos_marker); + asm.push_insn(insn); + }, + Insn::Cmp { left, right } | + Insn::Test { left, right } => { + let (split_left, split_right) = split_arithmetic_opnds(&mut asm, &live_ranges, index, &unmapped_opnds, left, right); + + *left = split_left; + *right = split_right; + + asm.push_insn(insn); }, // These instructions modify their input operand in-place, so we // may need to load the input value to preserve it - Insn { op: Op::LShift | Op::RShift | Op::URShift, opnds, target, text, pos_marker, .. } => { - let (opnd0, opnd1) = match (opnds[0], opnds[1]) { + Insn::LShift { opnd, shift, out } | + Insn::RShift { opnd, shift, out } | + Insn::URShift { opnd, shift, out } => { + match (&unmapped_opnds[0], &unmapped_opnds[1]) { // Instruction output whose live range spans beyond this instruction (Opnd::InsnOut { idx, .. }, _) => { - if live_ranges[idx] > index { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) - } else { - (mapped_opnds[0], mapped_opnds[1]) + if live_ranges[*idx] > index { + *opnd = asm.load(*opnd); } }, // We have to load memory operands to avoid corrupting them (Opnd::Mem(_) | Opnd::Reg(_), _) => { - (asm.load(mapped_opnds[0]), mapped_opnds[1]) + *opnd = asm.load(*opnd); }, - _ => (mapped_opnds[0], mapped_opnds[1]) + _ => {} }; - asm.push_insn_parts(insn.op, vec![opnd0, opnd1], target, text, pos_marker); - }, - Insn { op: Op::CSelZ | Op::CSelNZ | Op::CSelE | Op::CSelNE | Op::CSelL | Op::CSelLE | Op::CSelG | Op::CSelGE, target, text, pos_marker, .. } => { - let new_opnds = mapped_opnds.into_iter().map(|opnd| { - match opnd { - Opnd::Reg(_) | Opnd::InsnOut { .. } => opnd, - _ => asm.load(opnd) + *out = asm.next_opnd_out(Opnd::match_num_bits(&[*opnd, *shift])); + asm.push_insn(insn); + }, + Insn::CSelZ { truthy, falsy, out } | + Insn::CSelNZ { truthy, falsy, out } | + Insn::CSelE { truthy, falsy, out } | + Insn::CSelNE { truthy, falsy, out } | + Insn::CSelL { truthy, falsy, out } | + Insn::CSelLE { truthy, falsy, out } | + Insn::CSelG { truthy, falsy, out } | + Insn::CSelGE { truthy, falsy, out } => { + match truthy { + Opnd::Reg(_) | Opnd::InsnOut { .. } => {}, + _ => { + *truthy = asm.load(*truthy); } - }).collect(); + }; - asm.push_insn_parts(insn.op, new_opnds, target, text, pos_marker); + match falsy { + Opnd::Reg(_) | Opnd::InsnOut { .. } => {}, + _ => { + *falsy = asm.load(*falsy); + } + }; + + *out = asm.next_opnd_out(Opnd::match_num_bits(&[*truthy, *falsy])); + asm.push_insn(insn); }, - Insn { op: Op::Mov, .. } => { - match (mapped_opnds[0], mapped_opnds[1]) { + Insn::Mov { dest, src } => { + match (&dest, &src) { (Opnd::Mem(_), Opnd::Mem(_)) => { // We load opnd1 because for mov, opnd0 is the output - let opnd1 = asm.load(mapped_opnds[1]); - asm.mov(mapped_opnds[0], opnd1); + let opnd1 = asm.load(*src); + asm.mov(*dest, opnd1); }, (Opnd::Mem(_), Opnd::UImm(value)) => { // 32-bit values will be sign-extended - if imm_num_bits(value as i64) > 32 { - let opnd1 = asm.load(mapped_opnds[1]); - asm.mov(mapped_opnds[0], opnd1); + if imm_num_bits(*value as i64) > 32 { + let opnd1 = asm.load(*src); + asm.mov(*dest, opnd1); } else { - asm.mov(mapped_opnds[0], mapped_opnds[1]); + asm.mov(*dest, *src); } }, (Opnd::Mem(_), Opnd::Imm(value)) => { - if imm_num_bits(value) > 32 { - let opnd1 = asm.load(mapped_opnds[1]); - asm.mov(mapped_opnds[0], opnd1); + if imm_num_bits(*value) > 32 { + let opnd1 = asm.load(*src); + asm.mov(*dest, opnd1); } else { - asm.mov(mapped_opnds[0], mapped_opnds[1]); + asm.mov(*dest, *src); } }, _ => { - asm.mov(mapped_opnds[0], mapped_opnds[1]); + asm.mov(*dest, *src); } } }, - Insn { op: Op::Not, opnds, .. } => { - let opnd0 = match opnds[0] { + Insn::Not { opnd, .. } => { + let opnd0 = match unmapped_opnds[0] { // If we have an instruction output whose live range // spans beyond this instruction, we have to load it. Opnd::InsnOut { idx, .. } => { if live_ranges[idx] > index { - asm.load(mapped_opnds[0]) + asm.load(*opnd) } else { - mapped_opnds[0] + *opnd } }, // We have to load memory and register operands to avoid // corrupting them. Opnd::Mem(_) | Opnd::Reg(_) => { - asm.load(mapped_opnds[0]) + asm.load(*opnd) }, // Otherwise we can just reuse the existing operand. - _ => mapped_opnds[0] + _ => *opnd }; asm.not(opnd0); }, _ => { - asm.push_insn_parts(insn.op, mapped_opnds, insn.target, insn.text, insn.pos_marker); + if insn.out_opnd().is_some() { + let out_num_bits = Opnd::match_num_bits_iter(insn.opnd_iter()); + let out = insn.out_opnd_mut().unwrap(); + *out = asm.next_opnd_out(out_num_bits); + } + + asm.push_insn(insn); } }; @@ -281,26 +338,24 @@ impl Assembler let start_write_pos = cb.get_write_pos(); for insn in &self.insns { match insn { - Insn { op: Op::Comment, text, .. } => { + Insn::Comment(text) => { if cfg!(feature = "asm_comments") { - cb.add_comment(text.as_ref().unwrap()); + cb.add_comment(text); } }, // Write the label at the current position - Insn { op: Op::Label, target, .. } => { - cb.write_label(target.unwrap().unwrap_label_idx()); + Insn::Label(target) => { + cb.write_label(target.unwrap_label_idx()); }, // Report back the current position in the generated code - Insn { op: Op::PosMarker, pos_marker, .. } => { - let pos = cb.get_write_ptr(); - let pos_marker_fn = pos_marker.as_ref().unwrap(); - pos_marker_fn(pos); + Insn::PosMarker(pos_marker) => { + pos_marker(cb.get_write_ptr()); }, - Insn { op: Op::BakeString, text, .. } => { - for byte in text.as_ref().unwrap().as_bytes() { + Insn::BakeString(text) => { + for byte in text.as_bytes() { cb.write_byte(*byte); } @@ -309,55 +364,55 @@ impl Assembler cb.write_byte(0); }, - Insn { op: Op::Add, opnds, .. } => { - add(cb, opnds[0].into(), opnds[1].into()) + Insn::Add { left, right, .. } => { + add(cb, left.into(), right.into()) }, - Insn { op: Op::FrameSetup, .. } => {}, - Insn { op: Op::FrameTeardown, .. } => {}, + Insn::FrameSetup => {}, + Insn::FrameTeardown => {}, - Insn { op: Op::Sub, opnds, .. } => { - sub(cb, opnds[0].into(), opnds[1].into()) + Insn::Sub { left, right, .. } => { + sub(cb, left.into(), right.into()) }, - Insn { op: Op::And, opnds, .. } => { - and(cb, opnds[0].into(), opnds[1].into()) + Insn::And { left, right, .. } => { + and(cb, left.into(), right.into()) }, - Insn { op: Op::Or, opnds, .. } => { - or(cb, opnds[0].into(), opnds[1].into()); + Insn::Or { left, right, .. } => { + or(cb, left.into(), right.into()); }, - Insn { op: Op::Xor, opnds, .. } => { - xor(cb, opnds[0].into(), opnds[1].into()); + Insn::Xor { left, right, .. } => { + xor(cb, left.into(), right.into()); }, - Insn { op: Op::Not, opnds, .. } => { - not(cb, opnds[0].into()); + Insn::Not { opnd, .. } => { + not(cb, opnd.into()); }, - Insn { op: Op::LShift, opnds, .. } => { - shl(cb, opnds[0].into(), opnds[1].into()) + Insn::LShift { opnd, shift , ..} => { + shl(cb, opnd.into(), shift.into()) }, - Insn { op: Op::RShift, opnds, .. } => { - sar(cb, opnds[0].into(), opnds[1].into()) + Insn::RShift { opnd, shift , ..} => { + sar(cb, opnd.into(), shift.into()) }, - Insn { op: Op::URShift, opnds, .. } => { - shr(cb, opnds[0].into(), opnds[1].into()) + Insn::URShift { opnd, shift, .. } => { + shr(cb, opnd.into(), shift.into()) }, - Insn { op: Op::Store, opnds, .. } => { - mov(cb, opnds[0].into(), opnds[1].into()); + Insn::Store { dest, src } => { + mov(cb, dest.into(), src.into()); }, // This assumes only load instructions can contain references to GC'd Value operands - Insn { op: Op::Load, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); + Insn::Load { opnd, out } => { + mov(cb, out.into(), opnd.into()); // If the value being loaded is a heap object - if let Opnd::Value(val) = opnds[0] { + if let Opnd::Value(val) = opnd { if !val.special_const_p() { // The pointer immediate is encoded as the last part of the mov written out let ptr_offset: u32 = (cb.get_write_pos() as u32) - (SIZEOF_VALUE as u32); @@ -366,45 +421,45 @@ impl Assembler } }, - Insn { op: Op::LoadSExt, opnds, out, .. } => { - movsx(cb, (*out).into(), opnds[0].into()); + Insn::LoadSExt { opnd, out } => { + movsx(cb, out.into(), opnd.into()); }, - Insn { op: Op::Mov, opnds, .. } => { - mov(cb, opnds[0].into(), opnds[1].into()); + Insn::Mov { dest, src } => { + mov(cb, dest.into(), src.into()); }, // Load effective address - Insn { op: Op::Lea, opnds, out, .. } => { - lea(cb, (*out).into(), opnds[0].into()); + Insn::Lea { opnd, out } => { + lea(cb, out.into(), opnd.into()); }, // Load relative address - Insn { op: Op::LeaLabel, out, target, .. } => { - let label_idx = target.unwrap().unwrap_label_idx(); + Insn::LeaLabel { target, out } => { + let label_idx = target.unwrap_label_idx(); cb.label_ref(label_idx, 7, |cb, src_addr, dst_addr| { let disp = dst_addr - src_addr; lea(cb, Self::SCRATCH0, mem_opnd(8, RIP, disp.try_into().unwrap())); }); - mov(cb, (*out).into(), Self::SCRATCH0); + mov(cb, out.into(), Self::SCRATCH0); }, // Push and pop to/from the C stack - Insn { op: Op::CPush, opnds, .. } => { - push(cb, opnds[0].into()); + Insn::CPush(opnd) => { + push(cb, opnd.into()); }, - Insn { op: Op::CPop, out, .. } => { - pop(cb, (*out).into()); + Insn::CPop { out } => { + pop(cb, out.into()); }, - Insn { op: Op::CPopInto, opnds, .. } => { - pop(cb, opnds[0].into()); + Insn::CPopInto(opnd) => { + pop(cb, opnd.into()); }, // Push and pop to the C stack all caller-save registers and the // flags - Insn { op: Op::CPushAll, .. } => { + Insn::CPushAll => { let regs = Assembler::get_caller_save_regs(); for reg in regs { @@ -412,7 +467,7 @@ impl Assembler } pushfq(cb); }, - Insn { op: Op::CPopAll, .. } => { + Insn::CPopAll => { let regs = Assembler::get_caller_save_regs(); popfq(cb); @@ -422,7 +477,7 @@ impl Assembler }, // C function call - Insn { op: Op::CCall, opnds, target, .. } => { + Insn::CCall { opnds, target, .. } => { // Temporary assert!(opnds.len() <= _C_ARG_OPNDS.len()); @@ -431,92 +486,92 @@ impl Assembler mov(cb, X86Opnd::Reg(_C_ARG_OPNDS[idx].unwrap_reg()), opnds[idx].into()); } - let ptr = target.unwrap().unwrap_fun_ptr(); + let ptr = target.unwrap_fun_ptr(); call_ptr(cb, RAX, ptr); }, - Insn { op: Op::CRet, opnds, .. } => { + Insn::CRet(opnd) => { // TODO: bias allocation towards return register - if opnds[0] != Opnd::Reg(C_RET_REG) { - mov(cb, RAX, opnds[0].into()); + if *opnd != Opnd::Reg(C_RET_REG) { + mov(cb, RAX, opnd.into()); } ret(cb); }, // Compare - Insn { op: Op::Cmp, opnds, .. } => { - cmp(cb, opnds[0].into(), opnds[1].into()); + Insn::Cmp { left, right } => { + cmp(cb, left.into(), right.into()); } // Test and set flags - Insn { op: Op::Test, opnds, .. } => { - test(cb, opnds[0].into(), opnds[1].into()); + Insn::Test { left, right } => { + test(cb, left.into(), right.into()); } - Insn { op: Op::JmpOpnd, opnds, .. } => { - jmp_rm(cb, opnds[0].into()); + Insn::JmpOpnd(opnd) => { + jmp_rm(cb, opnd.into()); } // Conditional jump to a label - Insn { op: Op::Jmp, target, .. } => { - match target.unwrap() { + Insn::Jmp(target) => { + match *target { Target::CodePtr(code_ptr) => jmp_ptr(cb, code_ptr), Target::Label(label_idx) => jmp_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Je, target, .. } => { - match target.unwrap() { + Insn::Je(target) => { + match *target { Target::CodePtr(code_ptr) => je_ptr(cb, code_ptr), Target::Label(label_idx) => je_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Jne, target, .. } => { - match target.unwrap() { + Insn::Jne(target) => { + match *target { Target::CodePtr(code_ptr) => jne_ptr(cb, code_ptr), Target::Label(label_idx) => jne_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Jl, target, .. } => { - match target.unwrap() { + Insn::Jl(target) => { + match *target { Target::CodePtr(code_ptr) => jl_ptr(cb, code_ptr), Target::Label(label_idx) => jl_label(cb, label_idx), _ => unreachable!() } }, - Insn { op: Op::Jbe, target, .. } => { - match target.unwrap() { + Insn::Jbe(target) => { + match *target { Target::CodePtr(code_ptr) => jbe_ptr(cb, code_ptr), Target::Label(label_idx) => jbe_label(cb, label_idx), _ => unreachable!() } }, - Insn { op: Op::Jz, target, .. } => { - match target.unwrap() { + Insn::Jz(target) => { + match *target { Target::CodePtr(code_ptr) => jz_ptr(cb, code_ptr), Target::Label(label_idx) => jz_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Jnz, target, .. } => { - match target.unwrap() { + Insn::Jnz(target) => { + match *target { Target::CodePtr(code_ptr) => jnz_ptr(cb, code_ptr), Target::Label(label_idx) => jnz_label(cb, label_idx), _ => unreachable!() } } - Insn { op: Op::Jo, target, .. } => { - match target.unwrap() { + Insn::Jo(target) => { + match *target { Target::CodePtr(code_ptr) => jo_ptr(cb, code_ptr), Target::Label(label_idx) => jo_label(cb, label_idx), _ => unreachable!() @@ -524,49 +579,49 @@ impl Assembler } // Atomically increment a counter at a given memory location - Insn { op: Op::IncrCounter, opnds, .. } => { - assert!(matches!(opnds[0], Opnd::Mem(_))); - assert!(matches!(opnds[1], Opnd::UImm(_) | Opnd::Imm(_) ) ); + Insn::IncrCounter { mem, value } => { + assert!(matches!(mem, Opnd::Mem(_))); + assert!(matches!(value, Opnd::UImm(_) | Opnd::Imm(_) ) ); write_lock_prefix(cb); - add(cb, opnds[0].into(), opnds[1].into()); + add(cb, mem.into(), value.into()); }, - Insn { op: Op::Breakpoint, .. } => int3(cb), + Insn::Breakpoint => int3(cb), - Insn { op: Op::CSelZ, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovnz(cb, (*out).into(), opnds[1].into()); + Insn::CSelZ { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovnz(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelNZ, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovz(cb, (*out).into(), opnds[1].into()); + Insn::CSelNZ { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovz(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelE, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovne(cb, (*out).into(), opnds[1].into()); + Insn::CSelE { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovne(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelNE, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmove(cb, (*out).into(), opnds[1].into()); + Insn::CSelNE { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmove(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelL, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovge(cb, (*out).into(), opnds[1].into()); + Insn::CSelL { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovge(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelLE, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovg(cb, (*out).into(), opnds[1].into()); + Insn::CSelLE { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovg(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelG, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovle(cb, (*out).into(), opnds[1].into()); + Insn::CSelG { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovle(cb, out.into(), falsy.into()); }, - Insn { op: Op::CSelGE, opnds, out, .. } => { - mov(cb, (*out).into(), opnds[0].into()); - cmovl(cb, (*out).into(), opnds[1].into()); + Insn::CSelGE { truthy, falsy, out } => { + mov(cb, out.into(), truthy.into()); + cmovl(cb, out.into(), falsy.into()); } - Insn { op: Op::LiveReg, .. } => (), // just a reg alloc signal, no code - Insn { op: Op::PadEntryExit, .. } => { + Insn::LiveReg { .. } => (), // just a reg alloc signal, no code + Insn::PadEntryExit => { // We assume that our Op::Jmp usage that gets invalidated is <= 5 let code_size: u32 = (cb.get_write_pos() - start_write_pos).try_into().unwrap(); if code_size < 5 { @@ -578,7 +633,7 @@ impl Assembler // we feed to the backend could get lowered into other // instructions. So it's possible that some of our backend // instructions can never make it to the emit stage. - _ => panic!("unsupported instruction passed to x86 backend: {:?}", insn.op) + _ => panic!("unsupported instruction passed to x86 backend: {:?}", insn) }; } |