github
/
ruby
зеркало из https://github.com/github/ruby.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

YJIT: Introduce Target::SideExit (#7712) 

* YJIT: Introduce Target::SideExit

* YJIT: Obviate Insn::SideExitContext

* YJIT: Avoid cloning a Context for each insn
This commit is contained in:
Takashi Kokubun 2023-04-14 14:00:10 -07:00 коммит произвёл GitHub
Родитель d83e59e6b8
Коммит 4501fb8b46
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
9 изменённых файлов: 374 добавлений и 291 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

55
yjit/src/backend/arm64/mod.rs
Просмотреть файл

@ -2,8 +2,10 @@
#![allow(unused_variables)]
#![allow(unused_imports)]
use std::mem::take;
use crate::asm::x86_64::jmp_ptr;
use crate::asm::{CodeBlock};
use crate::asm::{CodeBlock, OutlinedCb};
use crate::asm::arm64::*;
use crate::codegen::{JITState, CodegenGlobals};
use crate::core::Context;
@ -374,7 +376,7 @@ impl Assembler
}
}
let mut asm_local = Assembler::new_with_label_names(std::mem::take(&mut self.label_names));
let mut asm_local = Assembler::new_with_label_names(take(&mut self.label_names), take(&mut self.side_exits));
let asm = &mut asm_local;
let mut iterator = self.into_draining_iter();
@ -675,7 +677,7 @@ impl Assembler
/// Emit platform-specific machine code
/// Returns a list of GC offsets. Can return failure to signal caller to retry.
fn arm64_emit(&mut self, cb: &mut CodeBlock) -> Result<Vec<u32>, ()> {
fn arm64_emit(&mut self, cb: &mut CodeBlock, ocb: &mut Option<&mut OutlinedCb>) -> Result<Vec<u32>, ()> {
/// Determine how many instructions it will take to represent moving
/// this value into a register. Note that the return value of this
/// function must correspond to how many instructions are used to
@ -765,6 +767,9 @@ impl Assembler
bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes));
});
},
Target::SideExit { .. } => {
unreachable!("Target::SideExit should have been compiled by compile_side_exit")
},
};
}
@ -780,6 +785,20 @@ impl Assembler
ldr_post(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP));
}
/// Compile a side exit if Target::SideExit is given.
fn compile_side_exit(
target: Target,
asm: &mut Assembler,
ocb: &mut Option<&mut OutlinedCb>,
) -> Target {
if let Target::SideExit { counter, context } = target {
let side_exit = asm.get_side_exit(&context.unwrap(), counter, ocb.as_mut().unwrap());
Target::SideExitPtr(side_exit)
} else {
target
}
}
// dbg!(&self.insns);
// List of GC offsets
@ -1016,12 +1035,12 @@ impl Assembler
br(cb, opnd.into());
},
Insn::Jmp(target) => {
match target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(dst_ptr) => {
emit_jmp_ptr(cb, *dst_ptr, true);
emit_jmp_ptr(cb, dst_ptr, true);
},
Target::SideExitPtr(dst_ptr) => {
emit_jmp_ptr(cb, *dst_ptr, false);
emit_jmp_ptr(cb, dst_ptr, false);
},
Target::Label(label_idx) => {
// Here we're going to save enough space for
@ -1029,27 +1048,30 @@ impl Assembler
// instruction once we know the offset. We're going
// to assume we can fit into a single b instruction.
// It will panic otherwise.
cb.label_ref(*label_idx, 4, |cb, src_addr, dst_addr| {
cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| {
let bytes: i32 = (dst_addr - (src_addr - 4)).try_into().unwrap();
b(cb, InstructionOffset::from_bytes(bytes));
});
},
Target::SideExit { .. } => {
unreachable!("Target::SideExit should have been compiled by compile_side_exit")
},
};
},
Insn::Je(target) | Insn::Jz(target) => {
emit_conditional_jump::<{Condition::EQ}>(cb, *target);
emit_conditional_jump::<{Condition::EQ}>(cb, compile_side_exit(*target, self, ocb));
},
Insn::Jne(target) | Insn::Jnz(target) => {
emit_conditional_jump::<{Condition::NE}>(cb, *target);
emit_conditional_jump::<{Condition::NE}>(cb, compile_side_exit(*target, self, ocb));
},
Insn::Jl(target) => {
emit_conditional_jump::<{Condition::LT}>(cb, *target);
emit_conditional_jump::<{Condition::LT}>(cb, compile_side_exit(*target, self, ocb));
},
Insn::Jbe(target) => {
emit_conditional_jump::<{Condition::LS}>(cb, *target);
emit_conditional_jump::<{Condition::LS}>(cb, compile_side_exit(*target, self, ocb));
},
Insn::Jo(target) => {
emit_conditional_jump::<{Condition::VS}>(cb, *target);
emit_conditional_jump::<{Condition::VS}>(cb, compile_side_exit(*target, self, ocb));
},
Insn::IncrCounter { mem, value } => {
let label = cb.new_label("incr_counter_loop".to_string());
@ -1121,7 +1143,7 @@ impl Assembler
}
/// Optimize and compile the stored instructions
pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Vec<u32>
pub fn compile_with_regs(self, cb: &mut CodeBlock, ocb: Option<&mut OutlinedCb>, regs: Vec<Reg>) -> Vec<u32>
{
let asm = self.lower_stack();
let asm = asm.arm64_split();
@ -1135,14 +1157,15 @@ impl Assembler
let start_ptr = cb.get_write_ptr();
let starting_label_state = cb.get_label_state();
let gc_offsets = asm.arm64_emit(cb)
let mut ocb = ocb; // for &mut
let gc_offsets = asm.arm64_emit(cb, &mut ocb)
.unwrap_or_else(|_err| {
// we want to lower jumps to labels to b.cond instructions, which have a 1 MiB
// range limit. We can easily exceed the limit in case the jump straddles two pages.
// In this case, we retry with a fresh page.
cb.set_label_state(starting_label_state);
cb.next_page(start_ptr, emit_jmp_ptr_with_invalidation);
asm.arm64_emit(cb).expect("should not fail when writing to a fresh code page")
asm.arm64_emit(cb, &mut ocb).expect("should not fail when writing to a fresh code page")
});
if cb.has_dropped_bytes() {
@ -1180,7 +1203,7 @@ mod tests {
let opnd = asm.add(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG));
asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd);
asm.compile_with_regs(&mut cb, vec![X3_REG]);
asm.compile_with_regs(&mut cb, None, vec![X3_REG]);
// Assert that only 2 instructions were written.
assert_eq!(8, cb.get_write_pos());

115
yjit/src/backend/ir.rs
Просмотреть файл

@ -3,13 +3,15 @@
#![allow(unused_imports)]
use std::cell::Cell;
use std::collections::HashMap;
use std::fmt;
use std::convert::From;
use std::io::Write;
use std::mem::take;
use crate::codegen::{gen_outlined_exit, gen_counted_exit};
use crate::cruby::{VALUE, SIZEOF_VALUE_I32};
use crate::virtualmem::{CodePtr};
use crate::asm::{CodeBlock, uimm_num_bits, imm_num_bits};
use crate::asm::{CodeBlock, uimm_num_bits, imm_num_bits, OutlinedCb};
use crate::core::{Context, Type, TempMapping, RegTemps, MAX_REG_TEMPS, MAX_TEMP_TYPES};
use crate::options::*;
use crate::stats::*;
@ -280,13 +282,22 @@ impl From<VALUE> for Opnd {
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Target
{
CodePtr(CodePtr), // Pointer to a piece of YJIT-generated code
SideExitPtr(CodePtr), // Pointer to a side exit code
Label(usize), // A label within the generated code
/// Pointer to a piece of YJIT-generated code
CodePtr(CodePtr),
/// Side exit with a counter
SideExit { counter: Option<Counter>, context: Option<SideExitContext> },
/// Pointer to a side exit code
SideExitPtr(CodePtr),
/// A label within the generated code
Label(usize),
}
impl Target
{
pub fn side_exit(counter: Option<Counter>) -> Target {
Target::SideExit { counter, context: None }
}
pub fn unwrap_label_idx(&self) -> usize {
match self {
Target::Label(idx) => *idx,
@ -500,6 +511,25 @@ impl Insn {
InsnOpndMutIterator::new(self)
}
/// Get a mutable reference to a Target if it exists.
pub(super) fn target_mut(&mut self) -> Option<&mut Target> {
match self {
Insn::Jbe(target) |
Insn::Je(target) |
Insn::Jl(target) |
Insn::Jmp(target) |
Insn::Jne(target) |
Insn::Jnz(target) |
Insn::Jo(target) |
Insn::Jz(target) |
Insn::Label(target) |
Insn::LeaLabel { target, .. } => {
Some(target)
}
_ => None,
}
}
/// Returns a string that describes which operation this instruction is
/// performing. This is used for debugging.
fn op(&self) -> &'static str {
@ -880,10 +910,19 @@ impl fmt::Debug for Insn {
}
}
/// Set of variables used for generating side exits
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct SideExitContext {
/// PC of the instruction being compiled
pub pc: *mut VALUE,
/// Context when it started to compile the instruction
pub ctx: Context,
}
/// Object into which we assemble instructions to be
/// optimized and lowered
pub struct Assembler
{
pub struct Assembler {
pub(super) insns: Vec<Insn>,
/// Parallel vec with insns
@ -899,21 +938,33 @@ pub struct Assembler
/// Context for generating the current insn
pub ctx: Context,
/// Side exit caches for each SideExitContext
pub(super) side_exits: HashMap<SideExitContext, CodePtr>,
/// PC for Target::SideExit
side_exit_pc: Option<*mut VALUE>,
/// Stack size for Target::SideExit
side_exit_stack_size: Option<u8>,
}
impl Assembler
{
pub fn new() -> Self {
Self::new_with_label_names(Vec::default())
Self::new_with_label_names(Vec::default(), HashMap::default())
}
pub fn new_with_label_names(label_names: Vec<String>) -> Self {
pub fn new_with_label_names(label_names: Vec<String>, side_exits: HashMap<SideExitContext, CodePtr>) -> Self {
Self {
insns: Vec::default(),
live_ranges: Vec::default(),
reg_temps: Vec::default(),
label_names,
ctx: Context::default(),
side_exits,
side_exit_pc: None,
side_exit_stack_size: None,
}
}
@ -924,6 +975,12 @@ impl Assembler
regs.drain(0..num_regs).collect()
}
/// Set a context for generating side exits
pub fn set_side_exit_context(&mut self, pc: *mut VALUE, stack_size: u8) {
self.side_exit_pc = Some(pc);
self.side_exit_stack_size = Some(stack_size);
}
/// Build an Opnd::InsnOut from the current index of the assembler and the
/// given number of bits.
pub(super) fn next_opnd_out(&self, num_bits: u8) -> Opnd {
@ -973,6 +1030,18 @@ impl Assembler
}
}
// Set a side exit context to Target::SideExit
let mut insn = insn;
if let Some(Target::SideExit { context, .. }) = insn.target_mut() {
// We should skip this when this instruction is being copied from another Assembler.
if context.is_none() {
*context = Some(SideExitContext {
pc: self.side_exit_pc.unwrap(),
ctx: self.ctx.with_stack_size(self.side_exit_stack_size.unwrap()),
});
}
}
self.insns.push(insn);
self.live_ranges.push(insn_idx);
self.reg_temps.push(reg_temps);
@ -983,6 +1052,26 @@ impl Assembler
*self.reg_temps.last().unwrap_or(&RegTemps::default())
}
/// Get a cached side exit, wrapping a counter if specified
pub fn get_side_exit(&mut self, side_exit_context: &SideExitContext, counter: Option<Counter>, ocb: &mut OutlinedCb) -> CodePtr {
// Drop type information from a cache key
let mut side_exit_context = side_exit_context.clone();
side_exit_context.ctx = side_exit_context.ctx.get_generic_ctx();
// Get a cached side exit
let side_exit = match self.side_exits.get(&side_exit_context) {
None => {
let exit_code = gen_outlined_exit(side_exit_context.pc, &side_exit_context.ctx, ocb);
self.side_exits.insert(side_exit_context.clone(), exit_code);
exit_code
}
Some(code_ptr) => *code_ptr,
};
// Wrap a counter if needed
gen_counted_exit(side_exit, ocb, counter)
}
/// Create a new label instance that we can jump to
pub fn new_label(&mut self, name: &str) -> Target
{
@ -1016,7 +1105,7 @@ impl Assembler
}
}
let mut asm = Assembler::new_with_label_names(take(&mut self.label_names));
let mut asm = Assembler::new_with_label_names(take(&mut self.label_names), take(&mut self.side_exits));
let regs = Assembler::get_temp_regs();
let reg_temps = take(&mut self.reg_temps);
let mut iterator = self.into_draining_iter();
@ -1172,7 +1261,7 @@ impl Assembler
}
let live_ranges: Vec<usize> = take(&mut self.live_ranges);
let mut asm = Assembler::new_with_label_names(take(&mut self.label_names));
let mut asm = Assembler::new_with_label_names(take(&mut self.label_names), take(&mut self.side_exits));
let mut iterator = self.into_draining_iter();
while let Some((index, mut insn)) = iterator.next_unmapped() {
@ -1305,13 +1394,13 @@ impl Assembler
/// Compile the instructions down to machine code
/// NOTE: should compile return a list of block labels to enable
/// compiling multiple blocks at a time?
pub fn compile(self, cb: &mut CodeBlock) -> Vec<u32>
pub fn compile(self, cb: &mut CodeBlock, ocb: Option<&mut OutlinedCb>) -> Vec<u32>
{
#[cfg(feature = "disasm")]
let start_addr = cb.get_write_ptr();
let alloc_regs = Self::get_alloc_regs();
let gc_offsets = self.compile_with_regs(cb, alloc_regs);
let gc_offsets = self.compile_with_regs(cb, ocb, alloc_regs);
#[cfg(feature = "disasm")]
if let Some(dump_disasm) = get_option_ref!(dump_disasm) {
@ -1327,7 +1416,7 @@ impl Assembler
{
let mut alloc_regs = Self::get_alloc_regs();
let alloc_regs = alloc_regs.drain(0..num_regs).collect();
self.compile_with_regs(cb, alloc_regs)
self.compile_with_regs(cb, None, alloc_regs)
}
/// Consume the assembler by creating a new draining iterator.

2
yjit/src/backend/tests.rs
Просмотреть файл

@ -199,7 +199,7 @@ fn test_alloc_ccall_regs() {
let out2 = asm.ccall(0 as *const u8, vec![out1]);
asm.mov(EC, out2);
let mut cb = CodeBlock::new_dummy(1024);
asm.compile_with_regs(&mut cb, Assembler::get_alloc_regs());
asm.compile_with_regs(&mut cb, None, Assembler::get_alloc_regs());
}
#[test]

61
yjit/src/backend/x86_64/mod.rs
Просмотреть файл

@ -7,6 +7,7 @@ use std::mem::take;
use crate::asm::*;
use crate::asm::x86_64::*;
use crate::codegen::{JITState};
use crate::core::Context;
use crate::cruby::*;
use crate::backend::ir::*;
use crate::codegen::CodegenGlobals;
@ -115,7 +116,7 @@ impl Assembler
fn x86_split(mut self) -> Assembler
{
let live_ranges: Vec<usize> = take(&mut self.live_ranges);
let mut asm = Assembler::new_with_label_names(take(&mut self.label_names));
let mut asm = Assembler::new_with_label_names(take(&mut self.label_names), take(&mut self.side_exits));
let mut iterator = self.into_draining_iter();
while let Some((index, mut insn)) = iterator.next_unmapped() {
@ -381,7 +382,7 @@ impl Assembler
}
/// Emit platform-specific machine code
pub fn x86_emit(&mut self, cb: &mut CodeBlock) -> Vec<u32>
pub fn x86_emit(&mut self, cb: &mut CodeBlock, ocb: &mut Option<&mut OutlinedCb>) -> Vec<u32>
{
/// For some instructions, we want to be able to lower a 64-bit operand
/// without requiring more registers to be available in the register
@ -411,6 +412,19 @@ impl Assembler
}
}
/// Compile a side exit if Target::SideExit is given.
fn compile_side_exit(
target: Target,
asm: &mut Assembler,
ocb: &mut Option<&mut OutlinedCb>,
) -> Target {
if let Target::SideExit { counter, context } = target {
let side_exit = asm.get_side_exit(&context.unwrap(), counter, ocb.as_mut().unwrap());
Target::SideExitPtr(side_exit)
} else {
target
}
}
fn emit_csel(cb: &mut CodeBlock, truthy: Opnd, falsy: Opnd, out: Opnd, cmov_fn: fn(&mut CodeBlock, X86Opnd, X86Opnd)) {
if out != truthy {
@ -426,8 +440,8 @@ impl Assembler
// For each instruction
let start_write_pos = cb.get_write_pos();
let mut insns_idx: usize = 0;
while let Some(insn) = self.insns.get(insns_idx) {
let mut insn_idx: usize = 0;
while let Some(insn) = self.insns.get(insn_idx) {
let src_ptr = cb.get_write_ptr();
let had_dropped_bytes = cb.has_dropped_bytes();
let old_label_state = cb.get_label_state();
@ -626,58 +640,66 @@ impl Assembler
// Conditional jump to a label
Insn::Jmp(target) => {
match *target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jmp_ptr(cb, code_ptr),
Target::Label(label_idx) => jmp_label(cb, label_idx),
Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exit"),
}
}
Insn::Je(target) => {
match *target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => je_ptr(cb, code_ptr),
Target::Label(label_idx) => je_label(cb, label_idx),
Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exit"),
}
}
Insn::Jne(target) => {
match *target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jne_ptr(cb, code_ptr),
Target::Label(label_idx) => jne_label(cb, label_idx),
Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exit"),
}
}
Insn::Jl(target) => {
match *target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jl_ptr(cb, code_ptr),
Target::Label(label_idx) => jl_label(cb, label_idx),
Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exit"),
}
},
Insn::Jbe(target) => {
match *target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jbe_ptr(cb, code_ptr),
Target::Label(label_idx) => jbe_label(cb, label_idx),
Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exit"),
}
},
Insn::Jz(target) => {
match *target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jz_ptr(cb, code_ptr),
Target::Label(label_idx) => jz_label(cb, label_idx),
Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exit"),
}
}
Insn::Jnz(target) => {
match *target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jnz_ptr(cb, code_ptr),
Target::Label(label_idx) => jnz_label(cb, label_idx),
Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exit"),
}
}
Insn::Jo(target) => {
match *target {
match compile_side_exit(*target, self, ocb) {
Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jo_ptr(cb, code_ptr),
Target::Label(label_idx) => jo_label(cb, label_idx),
Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exit"),
}
}
@ -724,13 +746,6 @@ impl Assembler
nop(cb, (cb.jmp_ptr_bytes() - code_size) as u32);
}
}
// We want to keep the panic here because some instructions that
// we feed to the backend could get lowered into other
// instructions. So it's possible that some of our backend
// instructions can never make it to the emit stage.
#[allow(unreachable_patterns)]
_ => panic!("unsupported instruction passed to x86 backend: {:?}", insn)
};
// On failure, jump to the next page and retry the current insn
@ -738,7 +753,7 @@ impl Assembler
// Reset cb states before retrying the current Insn
cb.set_label_state(old_label_state);
} else {
insns_idx += 1;
insn_idx += 1;
gc_offsets.append(&mut insn_gc_offsets);
}
}
@ -747,8 +762,7 @@ impl Assembler
}
/// Optimize and compile the stored instructions
pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Vec<u32>
{
pub fn compile_with_regs(self, cb: &mut CodeBlock, ocb: Option<&mut OutlinedCb>, regs: Vec<Reg>) -> Vec<u32> {
let asm = self.lower_stack();
let asm = asm.x86_split();
let mut asm = asm.alloc_regs(regs);
@ -759,7 +773,8 @@ impl Assembler
assert!(label_idx == idx);
}
let gc_offsets = asm.x86_emit(cb);
let mut ocb = ocb; // for &mut
let gc_offsets = asm.x86_emit(cb, &mut ocb);
if cb.has_dropped_bytes() {
cb.clear_labels();

335
yjit/src/codegen.rs
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

78
yjit/src/core.rs
Просмотреть файл

@ -38,7 +38,7 @@ const MAX_LOCAL_TYPES: usize = 8;
pub type IseqIdx = u16;
// Represent the type of a value (local/stack/self) in YJIT
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
pub enum Type {
Unknown,
UnknownImm,
@ -298,7 +298,7 @@ pub enum TypeDiff {
// Potential mapping of a value on the temporary stack to
// self, a local variable or constant so that we can track its type
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
#[derive(Copy, Clone, Eq, Hash, PartialEq, Debug)]
pub enum TempMapping {
MapToStack, // Normal stack value
MapToSelf, // Temp maps to the self operand
@ -307,7 +307,7 @@ pub enum TempMapping {
}
// Index used by MapToLocal. Using this instead of u8 makes TempMapping 1 byte.
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
#[derive(Copy, Clone, Eq, Hash, PartialEq, Debug)]
pub enum LocalIndex {
Local0,
Local1,
@ -417,7 +417,7 @@ impl RegTemps {
/// Code generation context
/// Contains information we can use to specialize/optimize code
/// There are a lot of context objects so we try to keep the size small.
#[derive(Clone, Default, PartialEq, Debug)]
#[derive(Clone, Copy, Default, Eq, Hash, PartialEq, Debug)]
pub struct Context {
// Number of values currently on the temporary stack
stack_size: u8,
@ -478,12 +478,12 @@ pub enum BranchGenFn {
}
impl BranchGenFn {
pub fn call(&self, asm: &mut Assembler, target0: CodePtr, target1: Option<CodePtr>) {
pub fn call(&self, asm: &mut Assembler, target0: Target, target1: Option<Target>) {
match self {
BranchGenFn::BranchIf(shape) => {
match shape.get() {
BranchShape::Next0 => asm.jz(target1.unwrap().into()),
BranchShape::Next1 => asm.jnz(target0.into()),
BranchShape::Next0 => asm.jz(target1.unwrap()),
BranchShape::Next1 => asm.jnz(target0),
BranchShape::Default => {
asm.jnz(target0.into());
asm.jmp(target1.unwrap().into());
@ -492,21 +492,21 @@ impl BranchGenFn {
}
BranchGenFn::BranchNil(shape) => {
match shape.get() {
BranchShape::Next0 => asm.jne(target1.unwrap().into()),
BranchShape::Next1 => asm.je(target0.into()),
BranchShape::Next0 => asm.jne(target1.unwrap()),
BranchShape::Next1 => asm.je(target0),
BranchShape::Default => {
asm.je(target0.into());
asm.jmp(target1.unwrap().into());
asm.je(target0);
asm.jmp(target1.unwrap());
}
}
}
BranchGenFn::BranchUnless(shape) => {
match shape.get() {
BranchShape::Next0 => asm.jnz(target1.unwrap().into()),
BranchShape::Next1 => asm.jz(target0.into()),
BranchShape::Next0 => asm.jnz(target1.unwrap()),
BranchShape::Next1 => asm.jz(target0),
BranchShape::Default => {
asm.jz(target0.into());
asm.jmp(target1.unwrap().into());
asm.jz(target0);
asm.jmp(target1.unwrap());
}
}
}
@ -522,14 +522,14 @@ impl BranchGenFn {
asm.jnz(target0.into())
}
BranchGenFn::JZToTarget0 => {
asm.jz(Target::CodePtr(target0))
asm.jz(target0)
}
BranchGenFn::JBEToTarget0 => {
asm.jbe(Target::CodePtr(target0))
asm.jbe(target0)
}
BranchGenFn::JITReturn => {
asm.comment("update cfp->jit_return");
asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(target0.raw_ptr()));
asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(target0.unwrap_code_ptr().raw_ptr()));
}
}
}
@ -1378,10 +1378,7 @@ pub fn limit_block_versions(blockid: BlockId, ctx: &Context) -> Context {
// Produce a generic context that stores no type information,
// but still respects the stack_size and sp_offset constraints.
// This new context will then match all future requests.
let mut generic_ctx = Context::default();
generic_ctx.stack_size = ctx.stack_size;
generic_ctx.sp_offset = ctx.sp_offset;
generic_ctx.reg_temps = ctx.reg_temps;
let generic_ctx = ctx.get_generic_ctx();
debug_assert_ne!(
TypeDiff::Incompatible,
@ -1570,6 +1567,15 @@ impl Context {
self.stack_size
}
/// Create a new Context that is compatible with self but doesn't have type information.
pub fn get_generic_ctx(&self) -> Context {
let mut generic_ctx = Context::default();
generic_ctx.stack_size = self.stack_size;
generic_ctx.sp_offset = self.sp_offset;
generic_ctx.reg_temps = self.reg_temps;
generic_ctx
}
/// Create a new Context instance with a given stack_size and sp_offset adjusted
/// accordingly. This is useful when you want to virtually rewind a stack_size for
/// generating a side exit while considering past sp_offset changes on gen_save_sp.
@ -2170,7 +2176,7 @@ pub fn regenerate_entry(cb: &mut CodeBlock, entryref: &EntryRef, next_entry: Cod
let old_dropped_bytes = cb.has_dropped_bytes();
cb.set_write_ptr(unsafe { entryref.as_ref() }.start_addr);
cb.set_dropped_bytes(false);
asm.compile(cb);
asm.compile(cb, None);
// Rewind write_pos to the original one
assert_eq!(cb.get_write_ptr(), unsafe { entryref.as_ref() }.end_addr);
@ -2219,7 +2225,7 @@ fn entry_stub_hit_body(entry_ptr: *const c_void, ec: EcPtr) -> Option<*const u8>
let next_entry = cb.get_write_ptr();
let mut asm = Assembler::new();
let pending_entry = gen_entry_chain_guard(&mut asm, ocb, iseq, insn_idx)?;
asm.compile(cb);
asm.compile(cb, Some(ocb));
// Try to find an existing compiled version of this block
let blockid = BlockId { iseq, idx: insn_idx };
@ -2229,7 +2235,7 @@ fn entry_stub_hit_body(entry_ptr: *const c_void, ec: EcPtr) -> Option<*const u8>
Some(blockref) => {
let mut asm = Assembler::new();
asm.jmp(unsafe { blockref.as_ref() }.start_addr.into());
asm.compile(cb);
asm.compile(cb, Some(ocb));
blockref
}
// If this block hasn't yet been compiled, generate blocks after the entry guard.
@ -2273,7 +2279,7 @@ pub fn gen_entry_stub(entry_address: usize, ocb: &mut OutlinedCb) -> Option<Code
// Not really a side exit, just don't need a padded jump here.
asm.jmp(CodegenGlobals::get_entry_stub_hit_trampoline().as_side_exit());
asm.compile(ocb);
asm.compile(ocb, None);
if ocb.has_dropped_bytes() {
return None; // No space
@ -2296,7 +2302,7 @@ pub fn gen_entry_stub_hit_trampoline(ocb: &mut OutlinedCb) -> CodePtr {
// Jump to the address returned by the entry_stub_hit() call
asm.jmp_opnd(jump_addr);
asm.compile(ocb);
asm.compile(ocb, None);
code_ptr
}
@ -2316,8 +2322,8 @@ fn regenerate_branch(cb: &mut CodeBlock, branch: &Branch) {
asm.comment("regenerate_branch");
branch.gen_fn.call(
&mut asm,
branch.get_target_address(0).unwrap(),
branch.get_target_address(1),
Target::CodePtr(branch.get_target_address(0).unwrap()),
branch.get_target_address(1).map(|addr| Target::CodePtr(addr)),
);
// Rewrite the branch
@ -2325,7 +2331,7 @@ fn regenerate_branch(cb: &mut CodeBlock, branch: &Branch) {
let old_dropped_bytes = cb.has_dropped_bytes();
cb.set_write_ptr(branch.start_addr);
cb.set_dropped_bytes(false);
asm.compile(cb);
asm.compile(cb, None);
let new_end_addr = cb.get_write_ptr();
branch.end_addr.set(new_end_addr);
@ -2584,7 +2590,7 @@ fn gen_branch_stub(
// Not really a side exit, just don't need a padded jump here.
asm.jmp(CodegenGlobals::get_branch_stub_hit_trampoline().as_side_exit());
asm.compile(ocb);
asm.compile(ocb, None);
if ocb.has_dropped_bytes() {
// No space
@ -2623,7 +2629,7 @@ pub fn gen_branch_stub_hit_trampoline(ocb: &mut OutlinedCb) -> CodePtr {
// Jump to the address returned by the branch_stub_hit() call
asm.jmp_opnd(jump_addr);
asm.compile(ocb);
asm.compile(ocb, None);
code_ptr
}
@ -2715,7 +2721,7 @@ pub fn gen_branch(
// Call the branch generation function
asm.mark_branch_start(&branch);
if let Some(dst_addr) = target0_addr {
branch.gen_fn.call(asm, dst_addr, target1_addr);
branch.gen_fn.call(asm, Target::CodePtr(dst_addr), target1_addr.map(|addr| Target::CodePtr(addr)));
}
asm.mark_branch_end(&branch);
}
@ -2732,7 +2738,7 @@ pub fn gen_direct_jump(jit: &mut JITState, ctx: &Context, target0: BlockId, asm:
// Call the branch generation function
asm.comment("gen_direct_jmp: existing block");
asm.mark_branch_start(&branch);
branch.gen_fn.call(asm, block_addr, None);
branch.gen_fn.call(asm, Target::CodePtr(block_addr), None);
asm.mark_branch_end(&branch);
BranchTarget::Block(blockref)
@ -2787,7 +2793,7 @@ pub fn defer_compilation(
asm.comment("defer_compilation");
asm.mark_branch_start(&branch);
if let Some(dst_addr) = target0_address {
branch.gen_fn.call(asm, dst_addr, None);
branch.gen_fn.call(asm, Target::CodePtr(dst_addr), None);
}
asm.mark_branch_end(&branch);
@ -2962,7 +2968,7 @@ pub fn invalidate_block_version(blockref: &BlockRef) {
let mut asm = Assembler::new();
asm.jmp(block_entry_exit.as_side_exit());
cb.set_dropped_bytes(false);
asm.compile(&mut cb);
asm.compile(&mut cb, Some(ocb));
assert!(
cb.get_write_ptr() <= block_end,

13
yjit/src/invariants.rs
Просмотреть файл

@ -2,6 +2,7 @@
//! generated code if and when these assumptions are invalidated.
use crate::asm::OutlinedCb;
use crate::backend::ir::Assembler;
use crate::codegen::*;
use crate::core::*;
use crate::cruby::*;
@ -82,12 +83,13 @@ impl Invariants {
#[must_use]
pub fn assume_bop_not_redefined(
jit: &mut JITState,
asm: &mut Assembler,
ocb: &mut OutlinedCb,
klass: RedefinitionFlag,
bop: ruby_basic_operators,
) -> bool {
if unsafe { BASIC_OP_UNREDEFINED_P(bop, klass) } {
jit_ensure_block_entry_exit(jit, ocb);
jit_ensure_block_entry_exit(jit, asm, ocb);
jit.bop_assumptions.push((klass, bop));
return true;
@ -131,13 +133,14 @@ pub fn track_method_lookup_stability_assumption(
// default behavior.
pub fn assume_method_basic_definition(
jit: &mut JITState,
asm: &mut Assembler,
ocb: &mut OutlinedCb,
klass: VALUE,
mid: ID
) -> bool {
if unsafe { rb_method_basic_definition_p(klass, mid) } != 0 {
let cme = unsafe { rb_callable_method_entry(klass, mid) };
jit.assume_method_lookup_stable(ocb, cme);
jit.assume_method_lookup_stable(asm, ocb, cme);
true
} else {
false
@ -146,11 +149,11 @@ pub fn assume_method_basic_definition(
/// Tracks that a block is assuming it is operating in single-ractor mode.
#[must_use]
pub fn assume_single_ractor_mode(jit: &mut JITState, ocb: &mut OutlinedCb) -> bool {
pub fn assume_single_ractor_mode(jit: &mut JITState, asm: &mut Assembler, ocb: &mut OutlinedCb) -> bool {
if unsafe { rb_yjit_multi_ractor_p() } {
false
} else {
jit_ensure_block_entry_exit(jit, ocb);
jit_ensure_block_entry_exit(jit, asm, ocb);
jit.block_assumes_single_ractor = true;
true
@ -524,7 +527,7 @@ pub extern "C" fn rb_yjit_tracing_invalidate_all() {
cb.set_write_ptr(patch.inline_patch_pos);
cb.set_dropped_bytes(false);
asm.compile(cb);
asm.compile(cb, None);
last_patch_end = cb.get_write_ptr().raw_ptr();
}
cb.set_pos(old_pos);

2
yjit/src/stats.rs
Просмотреть файл

@ -135,7 +135,7 @@ macro_rules! make_counters {
/// Enum to represent a counter
#[allow(non_camel_case_types)]
#[derive(Clone, Copy)]
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Counter { $($counter_name),+ }
impl Counter {

4
yjit/src/utils.rs
Просмотреть файл

@ -263,7 +263,7 @@ mod tests {
let mut cb = CodeBlock::new_dummy(1024);
print_int(&mut asm, Opnd::Imm(42));
asm.compile(&mut cb);
asm.compile(&mut cb, None);
}
#[test]
@ -272,6 +272,6 @@ mod tests {
let mut cb = CodeBlock::new_dummy(1024);
print_str(&mut asm, "Hello, world!");
asm.compile(&mut cb);
asm.compile(&mut cb, None);
}
}