Skip to content

Commit

Permalink
Sized TrapIf
Browse files Browse the repository at this point in the history
  • Loading branch information
@MarinPostma
MarinPostma committed Dec 12, 2024
1 parent ec28731 commit 5ec6016
Show file tree
Hide file tree
Showing 31 changed files with 133 additions and 78 deletions.
1 change: 1 addition & 0 deletions cranelift/codegen/src/isa/aarch64/abi.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
// Here `Lo` == "less than" when interpreting the two
// operands as unsigned integers.
kind: CondBrKind::Cond(Cond::Lo),
size: OperandSize::Size64,
});
insts
}
Expand Down
29 changes: 15 additions & 14 deletions cranelift/codegen/src/isa/aarch64/inst.isle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -870,6 +870,7 @@
;; *execute* the embedded `Inst`. (In the emitted code, we use the inverse
;; of this condition in a branch that skips the trap instruction.)
(TrapIf
(size OperandSize)
(kind CondBrKind)
(trap_code TrapCode))

Expand Down Expand Up @@ -3506,7 +3507,7 @@
(side_effect
(with_flags_side_effect flags
(ConsumesFlags.ConsumesFlagsSideEffect
(MInst.TrapIf (cond_br_cond cond) trap_code)))))
(MInst.TrapIf (operand_size $I64) (cond_br_cond cond) trap_code)))))

;; Helpers for lowering `trapz` and `trapnz`.
(type ZeroCond
Expand All @@ -3526,7 +3527,7 @@
(let ((reg Reg (put_in_reg_zext64 val)))
(side_effect
(SideEffectNoResult.Inst
(MInst.TrapIf (zero_cond_to_cond_br zero_cond reg) trap_code)))))
(MInst.TrapIf (operand_size $I64) (zero_cond_to_cond_br zero_cond reg) trap_code)))))

(rule -1 (trap_if_val zero_cond val @ (value_type $I128) trap_code)
(let ((c ValueRegs (put_in_regs val))
Expand All @@ -3535,7 +3536,7 @@
(c_test Reg (orr $I64 c_lo c_hi)))
(side_effect
(SideEffectNoResult.Inst
(MInst.TrapIf (zero_cond_to_cond_br zero_cond c_test) trap_code)))))
(MInst.TrapIf (operand_size $I64) (zero_cond_to_cond_br zero_cond c_test) trap_code)))))

;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

Expand Down Expand Up @@ -3659,9 +3660,9 @@

;; Misc instruction helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(decl trap_if_zero_divisor (Reg) Reg)
(rule (trap_if_zero_divisor reg)
(let ((_ Unit (emit (MInst.TrapIf (cond_br_zero reg) (trap_code_division_by_zero)))))
(decl trap_if_zero_divisor (Reg OperandSize) Reg)
(rule (trap_if_zero_divisor reg size)
(let ((_ Unit (emit (MInst.TrapIf size (cond_br_zero reg) (trap_code_division_by_zero)))))
reg))

(decl size_from_ty (Type) OperandSize)
Expand All @@ -3685,7 +3686,7 @@
(u8_into_uimm5 1)
(nzcv false false false false)
(Cond.Eq))))
(_ Unit (emit (MInst.TrapIf (cond_br_cond (Cond.Vs))
(_ Unit (emit (MInst.TrapIf (operand_size $I64) (cond_br_cond (Cond.Vs))
(trap_code_integer_overflow))))
)
x))
Expand All @@ -3709,7 +3710,7 @@
(with_flags_reg
producer
(ConsumesFlags.ConsumesFlagsSideEffect
(MInst.TrapIf (cond_br_cond (Cond.Hs)) tc))))
(MInst.TrapIf (operand_size $I64) (cond_br_cond (Cond.Hs)) tc))))

(decl sink_atomic_load (Inst) Reg)
(rule (sink_atomic_load x @ (atomic_load _ addr))
Expand Down Expand Up @@ -4272,7 +4273,7 @@
(let ((r ValueRegs
(with_flags (fpu_cmp size src src)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Vs))
(MInst.TrapIf (operand_size $I64) (cond_br_cond (Cond.Vs))
(trap_code_bad_conversion_to_integer))
src))))
(value_regs_get r 0)))
Expand All @@ -4285,31 +4286,31 @@
(let ((r ValueRegs
(with_flags (fpu_cmp (ScalarSize.Size32) src min)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Le))
(MInst.TrapIf (operand_size $I64) (cond_br_cond (Cond.Le))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
(rule (fpu_to_int_underflow_check true $F64 (fits_in_32 out_ty) src min)
(let ((r ValueRegs
(with_flags (fpu_cmp (ScalarSize.Size64) src min)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Le))
(MInst.TrapIf (operand_size $I64) (cond_br_cond (Cond.Le))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
(rule -1 (fpu_to_int_underflow_check true in_ty _out_ty src min)
(let ((r ValueRegs
(with_flags (fpu_cmp (scalar_size in_ty) src min)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Lt))
(MInst.TrapIf (operand_size $I64) (cond_br_cond (Cond.Lt))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
(rule (fpu_to_int_underflow_check false in_ty _out_ty src min)
(let ((r ValueRegs
(with_flags (fpu_cmp (scalar_size in_ty) src min)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Le))
(MInst.TrapIf (operand_size $I64) (cond_br_cond (Cond.Le))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
Expand All @@ -4319,7 +4320,7 @@
(let ((r ValueRegs
(with_flags (fpu_cmp size src max)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Ge))
(MInst.TrapIf (operand_size $I64) (cond_br_cond (Cond.Ge))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
Expand Down
41 changes: 32 additions & 9 deletions cranelift/codegen/src/isa/aarch64/inst/emit.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,10 +164,21 @@ fn enc_cbr(op_31_24: u32, off_18_0: u32, op_4: u32, cond: u32) -> u32 {
(op_31_24 << 24) | (off_18_0 << 5) | (op_4 << 4) | cond
}

fn enc_conditional_br(taken: BranchTarget, kind: CondBrKind) -> u32 {
/// Set the size bit of an instruction.
fn enc_op_size(op: u32, size: OperandSize) -> u32 {
(op & !(1 << 31)) | (size.sf_bit() << 31)
}

fn enc_conditional_br(taken: BranchTarget, kind: CondBrKind, size: OperandSize) -> u32 {
match kind {
CondBrKind::Zero(reg) => enc_cmpbr(0b1_011010_0, taken.as_offset19_or_zero(), reg),
CondBrKind::NotZero(reg) => enc_cmpbr(0b1_011010_1, taken.as_offset19_or_zero(), reg),
CondBrKind::Zero(reg) => enc_op_size(
enc_cmpbr(0b0_011010_0, taken.as_offset19_or_zero(), reg),
size,
),
CondBrKind::NotZero(reg) => enc_op_size(
enc_cmpbr(0b0_011010_1, taken.as_offset19_or_zero(), reg),
size,
),
CondBrKind::Cond(c) => enc_cbr(0b01010100, taken.as_offset19_or_zero(), 0b0, c.bits()),
}
}
Expand Down Expand Up @@ -1613,6 +1624,7 @@ impl MachInstEmit for Inst {
sink.put4(enc_conditional_br(
BranchTarget::Label(again_label),
CondBrKind::NotZero(x24),
OperandSize::Size64,
));
sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);
}
Expand Down Expand Up @@ -1690,6 +1702,7 @@ impl MachInstEmit for Inst {
sink.put4(enc_conditional_br(
BranchTarget::Label(out_label),
CondBrKind::Cond(Cond::Ne),
OperandSize::Size64,
));
sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19);

Expand All @@ -1706,6 +1719,7 @@ impl MachInstEmit for Inst {
sink.put4(enc_conditional_br(
BranchTarget::Label(again_label),
CondBrKind::NotZero(x24),
OperandSize::Size64,
));
sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);

Expand Down Expand Up @@ -2814,6 +2828,7 @@ impl MachInstEmit for Inst {
sink.put4(enc_conditional_br(
BranchTarget::Label(else_label),
CondBrKind::Cond(cond),
OperandSize::Size64,
));
sink.use_label_at_offset(br_else_offset, else_label, LabelUse::Branch19);

Expand Down Expand Up @@ -2999,10 +3014,11 @@ impl MachInstEmit for Inst {
let cond_off = sink.cur_offset();
if let Some(l) = taken.as_label() {
sink.use_label_at_offset(cond_off, l, LabelUse::Branch19);
let inverted = enc_conditional_br(taken, kind.invert()).to_le_bytes();
let inverted =
enc_conditional_br(taken, kind.invert(), OperandSize::Size64).to_le_bytes();
sink.add_cond_branch(cond_off, cond_off + 4, l, &inverted[..]);
}
sink.put4(enc_conditional_br(taken, kind));
sink.put4(enc_conditional_br(taken, kind, OperandSize::Size64));

// Unconditional part next.
let uncond_off = sink.cur_offset();
Expand Down Expand Up @@ -3037,11 +3053,15 @@ impl MachInstEmit for Inst {
}
sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
}
&Inst::TrapIf { kind, trap_code } => {
&Inst::TrapIf {
kind,
trap_code,
size,
} => {
let label = sink.defer_trap(trap_code);
// condbr KIND, LABEL
let off = sink.cur_offset();
sink.put4(enc_conditional_br(BranchTarget::Label(label), kind));
sink.put4(enc_conditional_br(BranchTarget::Label(label), kind, size));
sink.use_label_at_offset(off, label, LabelUse::Branch19);
}
&Inst::IndirectBr { rn, .. } => {
Expand Down Expand Up @@ -3087,8 +3107,11 @@ impl MachInstEmit for Inst {
// the middle; we depend on hardcoded PC-rel addressing below.

// Branch to default when condition code from prior comparison indicates.
let br =
enc_conditional_br(BranchTarget::Label(default), CondBrKind::Cond(Cond::Hs));
let br = enc_conditional_br(
BranchTarget::Label(default),
CondBrKind::Cond(Cond::Hs),
OperandSize::Size64,
);

// No need to inform the sink's branch folding logic about this branch, because it
// will not be merged with any other branch, flipped, or elided (it is not preceded
Expand Down
18 changes: 18 additions & 0 deletions cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5901,6 +5901,7 @@ fn test_aarch64_binemit() {

insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::NotZero(xreg(8)),
},
Expand All @@ -5909,6 +5910,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Zero(xreg(8)),
},
Expand All @@ -5917,6 +5919,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Ne),
},
Expand All @@ -5925,6 +5928,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Eq),
},
Expand All @@ -5933,6 +5937,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Lo),
},
Expand All @@ -5941,6 +5946,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Hs),
},
Expand All @@ -5949,6 +5955,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Pl),
},
Expand All @@ -5957,6 +5964,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Mi),
},
Expand All @@ -5965,6 +5973,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Vc),
},
Expand All @@ -5973,6 +5982,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Vs),
},
Expand All @@ -5981,6 +5991,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Ls),
},
Expand All @@ -5989,6 +6000,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Hi),
},
Expand All @@ -5997,6 +6009,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Lt),
},
Expand All @@ -6005,6 +6018,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Ge),
},
Expand All @@ -6013,6 +6027,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Le),
},
Expand All @@ -6021,6 +6036,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Gt),
},
Expand All @@ -6029,6 +6045,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Nv),
},
Expand All @@ -6037,6 +6054,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
size: OperandSize::Size64,
trap_code: TrapCode::STACK_OVERFLOW,
kind: CondBrKind::Cond(Cond::Al),
},
Expand Down
5 changes: 3 additions & 2 deletions cranelift/codegen/src/isa/aarch64/inst/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2669,15 +2669,16 @@ impl Inst {
&Inst::Brk => "brk #0".to_string(),
&Inst::Udf { .. } => "udf #0xc11f".to_string(),
&Inst::TrapIf {
size,
ref kind,
trap_code,
} => match kind {
&CondBrKind::Zero(reg) => {
let reg = pretty_print_reg(reg);
let reg = pretty_print_reg_sized(reg, size);
format!("cbz {reg}, #trap={trap_code}")
}
&CondBrKind::NotZero(reg) => {
let reg = pretty_print_reg(reg);
let reg = pretty_print_reg_sized(reg, size);
format!("cbnz {reg}, #trap={trap_code}")
}
&CondBrKind::Cond(c) => {
Expand Down
Loading

0 comments on commit 5ec6016

Please # to comment.